From 0f6c6c16e61338e35a29871ab0fb2296408f177a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 12 Apr 2022 19:22:20 +0200 Subject: [PATCH 001/139] outgoing messages --- node/overseer/overseer-gen/examples/dummy.rs | 14 +- .../proc-macro/src/impl_builder.rs | 3 +- .../proc-macro/src/impl_dispatch.rs | 4 +- .../proc-macro/src/impl_message_wrapper.rs | 2 +- .../src/{impl_misc.rs => impl_subsystem.rs} | 214 +++++++++++++----- .../overseer-gen/proc-macro/src/lib.rs | 6 +- .../proc-macro/src/parse_struct.rs | 121 ++++++++-- node/overseer/overseer-gen/src/lib.rs | 15 +- 8 files changed, 291 insertions(+), 88 deletions(-) rename node/overseer/overseer-gen/proc-macro/src/{impl_misc.rs => impl_subsystem.rs} (58%) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index d263d9fca887..f804f2906167 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -10,6 +10,7 @@ pub struct AwesomeSubSys; impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { fn start(self, _ctx: XxxSubsystemContext) -> SpawnedSubsystem { + self.spawn(async move { ctx.send_message(Plinko).await }); unimplemented!("starting yay!") } } @@ -19,6 +20,7 @@ pub struct GoblinTower; impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { fn start(self, _ctx: XxxSubsystemContext) -> SpawnedSubsystem { + self.spawn(async move { ctx.send_message(MsgStrukt).await }); unimplemented!("welcum") } } @@ -90,10 +92,18 @@ impl NetworkMsg { #[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] struct Xxx { - #[subsystem(MsgStrukt)] + #[subsystem( + handles: MsgStrukt, + sends: Plinko + )] sub0: AwesomeSubSys, - #[subsystem(no_dispatch, blocking, Plinko)] + #[subsystem(no_dispatch, blocking, + handles: [Plinko], + sends: [ + MsgStruckt + ]) + ] plinkos: GoblinTower, i_like_pi: f64, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 6d634cc9f344..4e9b96830e3b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -103,7 +103,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { info.subsystems().iter().filter(|ssf| !ssf.wip).enumerate().map(|(idx, ssf)| { let field_name = &ssf.name; let field_type = &ssf.generic; - let subsystem_consumes = &ssf.consumes; + let subsystem_consumes = &ssf.message_to_consume; // Remove state generic for the item to be replaced. It sufficient to know `field_type` for // that since we always move from `Init<#field_type>` to `Init`. let impl_subsystem_state_generics = recollect_without_idx(&subsystem_passthrough_state_generics[..], idx); @@ -155,6 +155,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { signal_capacity: self.signal_capacity, } } + /// Specify the the initialization function for a subsystem pub fn #field_name_with<'a, F>(self, subsystem_init_fn: F ) -> #builder diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 24cdd2ceed8b..ba1ba5294ce3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -35,7 +35,7 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> TokenStream { .into_iter() .filter(|ssf| !ssf.no_dispatch) .filter(|ssf| !ssf.wip) - .map(|ssf| ssf.consumes.clone()) + .map(|ssf| ssf.message_to_consume.clone()) .collect::>(); let mut ts = TokenStream::new(); @@ -49,7 +49,7 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> TokenStream { extern_msg // focuses on a `NetworkBridgeEvent< protocol_v1::* >` // TODO do not require this to be hardcoded, either externalize or ... - // https://github.com/paritytech/polkadot/issues/3427 + // .focus() .ok() .map(|event| { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 6de017e80055..0be039b78671 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -21,7 +21,7 @@ use super::*; /// Generates the wrapper type enum. pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result { - let consumes = info.consumes(); + let consumes = info.any_message(); let consumes_variant = info.variant_names(); let outgoing = &info.outgoing_ty; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs similarity index 58% rename from node/overseer/overseer-gen/proc-macro/src/impl_misc.rs rename to node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 4cc3dc6a974f..c7f522fa6042 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -14,25 +14,88 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use proc_macro2::TokenStream; use quote::quote; -use syn::Ident; +use syn::{Ident, Path, Result}; use super::*; -/// Implement a builder pattern for the `Overseer`-type, -/// which acts as the gateway to constructing the overseer. -pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { - let overseer_name = info.overseer_name.clone(); - let subsystem_sender_name = - Ident::new(&(overseer_name.to_string() + "SubsystemSender"), overseer_name.span()); - let subsystem_ctx_name = - Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span()); - let consumes = &info.consumes(); - let signal = &info.extern_signal_ty; - let wrapper_message = &info.message_wrapper; - let error_ty = &info.extern_error_ty; +pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { + let mut ts = TokenStream::new(); + + let span = info.overseer_name.span(); + let all_messages_wrapper = info.message_wrapper; let support_crate = info.support_crate_name(); + for ssf in info.subsystems() { + let subsystem_name = ssf.name.to_string(); + let subsystem_sender_name = Ident::new(&(subsystem_name + "SubsystemSender"), span); + let subsystem_ctx_name = Ident::new(&(subsystem_name + "SubsystemContext"), span); + + let outgoing_wrapper = Ident::new(&(subsystem_name + "OutgoingMessages"), span); + + ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?); + ts.extend(impl_subsystem_sender( + ssf, + &all_messages_wrapper, + support_crate, + &outgoing_wrapper, + &subsystem_sender_name, + &subsystem_ctx_name, + )); + ts.extend(impl_subsystem_context( + info, + &outgoing_wrapper, + &subsystem_sender_name, + &subsystem_ctx_name, + )); + } + Ok(ts) +} + +/// Generates the wrapper type enum, no bells or whistles. +pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Result { + // The message types are path based, each of them must finish with a type + // and as such we do this upfront. + let variants: Vec<_> = Result::from_iter(message_types.into_iter().map(|path| { + let x = path + .segments + .last() + .ok_or_else(|| { + syn::Error::new(wrapper.span(), "Path is empty, but it must end with an identifier") + }) + .map(|segment| segment.ident); + x + }))?; + let ts = quote! { + #[allow(missing_docs)] + #[derive(Clone)] + pub enum #wrapper { + #( + #variants ( #message_types ), + )* + } + + #( + impl ::std::convert::From< #message_types > for #wrapper { + fn from(message: #message_types) -> Self { + #wrapper :: #variants ( message ) + } + } + )* + }; + + Ok(ts) +} + +pub(crate) fn impl_subsystem_sender( + ssf: &SubSysField, + wrapper_message: &Ident, + support_crate: &TokenStream, + outgoing_wrapper: &Ident, + subsystem_sender_name: &Ident, + subsystem_ctx_name: &Ident, +) -> TokenStream { let ts = quote! { /// Connector to send messages towards all subsystems, /// while tracking the which signals where already received. @@ -43,57 +106,98 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { /// Systemwide tick for which signals were received by all subsystems. signals_received: SignalsReceived, } + }; - /// implementation for wrapping message type... - #[#support_crate ::async_trait] - impl SubsystemSender< #wrapper_message > for #subsystem_sender_name { - async fn send_message(&mut self, msg: #wrapper_message) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; - } + // Implement for all individual messages to avoid + // the necessity for manual wrapping, and to limit what a subsystem + // can actually send. This allows the generation of _the_ graph. + let sends = &ssf.messages_to_send; + ts.extend(quote!{ + #( + #[#support_crate ::async_trait] + impl SubsystemSender< #sends > for #subsystem_sender_name { + async fn send_message(&mut self, msg: #sends) { + self.channels.send_and_log_error( + self.signals_received.load(), + #wrapper_message ::from ( msg ) + ).await; + } - async fn send_messages(&mut self, msgs: T) - where - T: IntoIterator + Send, - T::IntoIter: Send, - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; + async fn send_messages(&mut self, msgs: T) + where + T: IntoIterator + Send, + T::IntoIter: Send, + { + // TODO This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } } - } - fn send_unbounded_message(&mut self, msg: #wrapper_message) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + fn send_unbounded_message(&mut self, msg: #sends) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )); + } } - } + )* + }); - // ... but also implement for all individual messages to avoid - // the necessity for manual wrapping, and do the conversion - // based on the generated `From::from` impl for the individual variants. - #( - #[#support_crate ::async_trait] - impl SubsystemSender< #consumes > for #subsystem_sender_name { - async fn send_message(&mut self, msg: #consumes) { - self.channels.send_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )).await; - } + // Create the same for a wrapping enum: + // + // 1. subsystem specific `*OutgoingMessages`-type + // 2. overseer-global-`AllMessages`-type + let wrapped = |wrapper: &Ident| { + quote! { + /// implementation for wrapping message type... + #[#support_crate ::async_trait] + impl SubsystemSender< #wrapper > for #subsystem_sender_name { + async fn send_message(&mut self, msg: #wrapper) { + self.channels.send_and_log_error(self.signals_received.load(), msg).await; + } - async fn send_messages(&mut self, msgs: T) - where - T: IntoIterator + Send, - T::IntoIter: Send, - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; + async fn send_messages(&mut self, msgs: T) + where + T: IntoIterator + Send, + T::IntoIter: Send, + { + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } } - } - fn send_unbounded_message(&mut self, msg: #consumes) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )); + fn send_unbounded_message(&mut self, msg: #wrapper) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + } } } - )* + }; + + // Allow the `#wrapper_message` to be sent as well. + ts.extend(wrapped(&outgoing_wrapper)); + // TODO FIXME + let inconsequent = true; + if inconsequent { + ts.extend(wrapped(wrapper_message)); + } + + ts +} + +/// Implement a builder pattern for the `Overseer`-type, +/// which acts as the gateway to constructing the overseer. +pub(crate) fn impl_subsystem_context( + info: &OverseerInfo, + outgoing_messages: &Ident, + subsystem_sender_name: &Ident, + subsystem_ctx_name: &Ident, +) -> TokenStream { + let signal = &info.extern_signal_ty; + let message_wrapper = &info.message_wrapper; + let error_ty = &info.extern_error_ty; + let support_crate = info.support_crate_name(); + + let ts = quote! { /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or to spawn it's [`SubsystemJob`]s. @@ -147,13 +251,13 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { #[#support_crate ::async_trait] impl #support_crate ::SubsystemContext for #subsystem_ctx_name where - #subsystem_sender_name: #support_crate ::SubsystemSender< #wrapper_message >, - #wrapper_message: From, + #subsystem_sender_name: #support_crate ::SubsystemSender< #outgoing_messages >, + #outgoing_messages: From, { type Message = M; type Signal = #signal; type Sender = #subsystem_sender_name; - type AllMessages = #wrapper_message; + type OutgoingMessages = #outgoing_messages; type Error = #error_ty; async fn try_recv(&mut self) -> ::std::result::Result>, ()> { diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index a0275e3e7987..b6bf19e8f401 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -24,8 +24,8 @@ mod impl_builder; mod impl_channels_out; mod impl_dispatch; mod impl_message_wrapper; -mod impl_misc; mod impl_overseer; +mod impl_subsystem; mod parse_attr; mod parse_struct; @@ -33,8 +33,8 @@ use impl_builder::*; use impl_channels_out::*; use impl_dispatch::*; use impl_message_wrapper::*; -use impl_misc::*; use impl_overseer::*; +use impl_subsystem::*; use parse_attr::*; use parse_struct::*; @@ -93,7 +93,7 @@ pub(crate) fn impl_overseer_gen( additive.extend(impl_overseen_subsystem(&info)); additive.extend(impl_channels_out_struct(&info)); - additive.extend(impl_misc(&info)); + additive.extend(impl_subsystem(&info)?); additive.extend(impl_message_wrapper_enum(&info)?); additive.extend(impl_dispatch(&info)); diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 2eed4001d6eb..51532e3dbb53 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -20,6 +20,7 @@ use syn::{ parse::{Parse, ParseStream}, punctuated::Punctuated, spanned::Spanned, + token::Bracket, AttrStyle, Attribute, Error, Field, FieldsNamed, GenericParam, Ident, ItemStruct, Path, Result, Token, Type, Visibility, }; @@ -30,6 +31,8 @@ mod kw { syn::custom_keyword!(wip); syn::custom_keyword!(no_dispatch); syn::custom_keyword!(blocking); + syn::custom_keyword!(consumes); + syn::custom_keyword!(sends); } #[derive(Clone, Debug)] @@ -43,6 +46,10 @@ enum SubSysAttrItem { /// External messages should not be - after being converted - /// be dispatched to the annotated subsystem. NoDispatch(kw::no_dispatch), + /// Message to be sent by this subsystem. + Sends(Sends), + /// Message to be consumed by this subsystem. + Consumes(Consumes), } impl Parse for SubSysAttrItem { @@ -54,6 +61,10 @@ impl Parse for SubSysAttrItem { Self::Blocking(input.parse::()?) } else if lookahead.peek(kw::no_dispatch) { Self::NoDispatch(input.parse::()?) + } else if lookahead.peek(kw::consumes) { + Self::Consumes(input.parse::()?) + } else if lookahead.peek(kw::sends) { + Self::Sends(input.parse::()?) } else { return Err(lookahead.error()) }) @@ -72,6 +83,12 @@ impl ToTokens for SubSysAttrItem { Self::NoDispatch(no_dispatch) => { quote! { #no_dispatch } }, + Self::Sends(_) => { + quote! {} + }, + Self::Consumes(_) => { + quote! {} + }, }; tokens.extend(ts.into_iter()); } @@ -87,8 +104,10 @@ pub(crate) struct SubSysField { /// which is also used `#wrapper_message :: #variant` variant /// part. pub(crate) generic: Ident, - /// Type to be consumed by the subsystem. - pub(crate) consumes: Path, + /// Type of message to be consumed by the subsystem. + pub(crate) message_to_consume: Path, + /// Types of messages to be sent by the subsystem. + pub(crate) messages_to_send: Vec, /// If `no_dispatch` is present, if the message is incoming via /// an `extern` `Event`, it will not be dispatched to all subsystems. pub(crate) no_dispatch: bool, @@ -115,6 +134,15 @@ macro_rules! extract_variant { ($unique:expr, $variant:ident ; err = $err:expr) => { extract_variant!($unique, $variant).ok_or_else(|| Error::new(Span::call_site(), $err)) }; + ($unique:expr, $variant:ident take) => { + $unique.values().find_map(|item| { + if let SubSysAttrItem::$variant(value) = item { + Some(value.clone()) + } else { + None + } + }) + }; ($unique:expr, $variant:ident) => { $unique.values().find_map(|item| { if let SubSysAttrItem::$variant(_) = item { @@ -126,6 +154,46 @@ macro_rules! extract_variant { }; } +#[derive(Debug, Clone)] +pub(crate) struct Sends { + pub(crate) keyword_sends: kw::sends, + pub(crate) colon: Token![:], + pub(crate) bracket: Bracket, + pub(crate) sends: Punctuated, +} + +impl Parse for Sends { + fn parse(input: syn::parse::ParseStream) -> Result { + let content; + Ok(Self { + keyword_sends: input.parse()?, + colon: input.parse()?, + bracket: syn::bracketed!(content in input), + sends: Punctuated::parse_terminated(&mut content)?, + }) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct Consumes { + pub(crate) keyword_consumes: kw::consumes, + pub(crate) colon: Token![:], + pub(crate) bracket: Bracket, + pub(crate) consumes: Punctuated, +} + +impl Parse for Consumes { + fn parse(input: syn::parse::ParseStream) -> Result { + let content; + Ok(Self { + keyword_consumes: input.parse()?, + colon: input.parse()?, + bracket: syn::bracketed!(content in input), + consumes: Punctuated::parse_terminated(&mut content)?, + }) + } +} + pub(crate) struct SubSystemTags { #[allow(dead_code)] pub(crate) attrs: Vec, @@ -134,12 +202,17 @@ pub(crate) struct SubSystemTags { /// The subsystem is in progress, only generate the `Wrapper` variant, but do not forward messages /// and also not include the subsystem in the list of subsystems. pub(crate) wip: bool, + /// If there are blocking components in the subsystem and hence it should be + /// spawned on a dedicated thread pool for such subssytems. pub(crate) blocking: bool, - pub(crate) consumes: Path, + /// The message type being consumed by the subsystem. + pub(crate) consumes: Option, + pub(crate) sends: Option, } impl Parse for SubSystemTags { fn parse(input: syn::parse::ParseStream) -> Result { + let span = input.span(); let attrs = Attribute::parse_outer(input)?; let input = input; @@ -154,29 +227,36 @@ impl Parse for SubSystemTags { assert!(items.empty_or_trailing(), "Always followed by the message type to consume. qed"); - let consumes = content.parse::()?; - let mut unique = HashMap::< std::mem::Discriminant, SubSysAttrItem, RandomState, >::default(); + for item in items { if let Some(first) = unique.insert(std::mem::discriminant(&item), item.clone()) { - let mut e = Error::new( - item.span(), - format!("Duplicate definition of subsystem attribute found"), - ); + let mut e = + Error::new(item.span(), "Duplicate definition of subsystem attribute found"); e.combine(Error::new(first.span(), "previously defined here.")); return Err(e) } } + // A subsystem makes no sense if not one of them is provided + let sends = extract_variant!(unique, Sends take); + let consumes = extract_variant!(unique, Consumes take); + if sends.is_none() && consumes.is_none() { + return Err(Error::new( + span, + "Must have at least one of `consumes: [..]` and `sends: [..]`.", + )) + } + let no_dispatch = extract_variant!(unique, NoDispatch; default = false); let blocking = extract_variant!(unique, Blocking; default = false); let wip = extract_variant!(unique, Wip; default = false); - Ok(Self { attrs, no_dispatch, blocking, consumes, wip }) + Ok(Self { attrs, no_dispatch, blocking, wip, sends, consumes }) } } @@ -220,7 +300,7 @@ pub(crate) struct OverseerInfo { pub(crate) extern_network_ty: Option, /// Type of messages that are sent to an external subsystem. - /// Merely here to be included during generation of `message_wrapper` type. + /// Merely here to be included during generation of `#message_wrapper` type. pub(crate) outgoing_ty: Option, /// Incoming event type from the outer world, commonly from the network. @@ -297,8 +377,8 @@ impl OverseerInfo { .collect::>() } - pub(crate) fn consumes(&self) -> Vec { - self.subsystems.iter().map(|ssf| ssf.consumes.clone()).collect::>() + pub(crate) fn any_message(&self) -> Vec { + self.subsystems.iter().map(|ssf| ssf.message_to_consume).collect::>() } pub(crate) fn channel_names_without_wip(&self, suffix: &'static str) -> Vec { @@ -313,7 +393,7 @@ impl OverseerInfo { self.subsystems .iter() .filter(|ssf| !ssf.wip) - .map(|ssf| ssf.consumes.clone()) + .map(|ssf| ssf.message_to_consume.clone()) .collect::>() } } @@ -365,11 +445,19 @@ impl OverseerGuts { let variant: SubSystemTags = syn::parse2(attr_tokens.clone())?; consumes_paths.push(variant.consumes); + let mut sends_paths = Vec::with_capacity(attrs.len()); + let attr_tokens = attr_tokens.clone(); + let variant: SubSystemTags = syn::parse2(attr_tokens.clone())?; + sends_paths.push(variant.consumes); + let field_ty = try_type_to_path(ty, span)?; let generic = field_ty .get_ident() .ok_or_else(|| { - Error::new(field_ty.span(), "Must be an identifier, not a path.") + Error::new( + field_ty.span(), + "Must be an identifier, not a path. It will be used as a generic.", + ) })? .clone(); if let Some(previous) = unique_subsystem_idents.get(&generic) { @@ -385,7 +473,8 @@ impl OverseerGuts { subsystems.push(SubSysField { name: ident, generic, - consumes: consumes_paths[0].clone(), + message_to_consume: consumes_paths[0].clone(), + messages_to_send: send_paths, no_dispatch: variant.no_dispatch, wip: variant.wip, blocking: variant.blocking, diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 0d926ead2daf..3184e88c2ca5 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -312,7 +312,7 @@ pub struct SubsystemMeterReadouts { /// /// [`Subsystem`]: trait.Subsystem.html /// -/// `M` here is the inner message type, and _not_ the generated `enum AllMessages`. +/// `M` here is the inner message type, and _not_ the generated `enum AllMessages` or `#message_wrapper` type. pub struct SubsystemInstance { /// Send sink for `Signal`s to be sent to a subsystem. pub tx_signal: crate::metered::MeteredSender, @@ -365,11 +365,10 @@ pub trait SubsystemContext: Send + 'static { type Message: std::fmt::Debug + Send + 'static; /// And the same for signals. type Signal: std::fmt::Debug + Send + 'static; - /// The overarching all messages `enum`. - /// In some cases can be identical to `Self::Message`. - type AllMessages: From + Send + 'static; + /// The overarching messages `enum` for this particular subsystem. + type OutgoingMessages: Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: SubsystemSender + Send + 'static; + type Sender: SubsystemSender + Send + 'static; /// The error type. type Error: ::std::error::Error + ::std::convert::From + Sync + Send + 'static; @@ -402,7 +401,7 @@ pub trait SubsystemContext: Send + 'static { Self::AllMessages: From, X: Send, { - self.sender().send_message(::from(msg)).await + self.sender().send_message(::from(msg)).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. @@ -410,7 +409,7 @@ pub trait SubsystemContext: Send + 'static { where T: IntoIterator + Send, T::IntoIter: Send, - Self::AllMessages: From, + Self::OutgoingMessages: From, X: Send, { self.sender() @@ -421,7 +420,7 @@ pub trait SubsystemContext: Send + 'static { /// Send a message using the unbounded connection. fn send_unbounded_message(&mut self, msg: X) where - Self::AllMessages: From, + Self::OutgoingMessages: From, X: Send, { self.sender().send_unbounded_message(Self::AllMessages::from(msg)) From 8fda133b14deebb920064df5b7c3d5c07e9a8bea Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 14 Apr 2022 19:59:04 +0200 Subject: [PATCH 002/139] refactor --- .../proc-macro/src/impl_subsystem.rs | 15 +- .../overseer-gen/proc-macro/src/lib.rs | 6 +- .../overseer-gen/proc-macro/src/parse/mod.rs | 24 ++ .../proc-macro/src/{ => parse}/parse_attr.rs | 0 .../src/{ => parse}/parse_struct.rs | 116 ++++--- .../proc-macro/src/parse/tests.rs | 303 ++++++++++++++++++ node/overseer/overseer-gen/src/lib.rs | 8 +- node/overseer/src/dummy.rs | 2 +- 8 files changed, 409 insertions(+), 65 deletions(-) create mode 100644 node/overseer/overseer-gen/proc-macro/src/parse/mod.rs rename node/overseer/overseer-gen/proc-macro/src/{ => parse}/parse_attr.rs (100%) rename node/overseer/overseer-gen/proc-macro/src/{ => parse}/parse_struct.rs (86%) create mode 100644 node/overseer/overseer-gen/proc-macro/src/parse/tests.rs diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index c7f522fa6042..843e754b5b40 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -24,15 +24,15 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let mut ts = TokenStream::new(); let span = info.overseer_name.span(); - let all_messages_wrapper = info.message_wrapper; + let all_messages_wrapper = &info.message_wrapper; let support_crate = info.support_crate_name(); for ssf in info.subsystems() { let subsystem_name = ssf.name.to_string(); - let subsystem_sender_name = Ident::new(&(subsystem_name + "SubsystemSender"), span); - let subsystem_ctx_name = Ident::new(&(subsystem_name + "SubsystemContext"), span); + let subsystem_sender_name = Ident::new(&(subsystem_name.clone() + "SubsystemSender"), span); + let subsystem_ctx_name = Ident::new(&(subsystem_name.clone() + "SubsystemContext"), span); - let outgoing_wrapper = Ident::new(&(subsystem_name + "OutgoingMessages"), span); + let outgoing_wrapper = Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span); ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?); ts.extend(impl_subsystem_sender( @@ -41,7 +41,6 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { support_crate, &outgoing_wrapper, &subsystem_sender_name, - &subsystem_ctx_name, )); ts.extend(impl_subsystem_context( info, @@ -64,7 +63,7 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu .ok_or_else(|| { syn::Error::new(wrapper.span(), "Path is empty, but it must end with an identifier") }) - .map(|segment| segment.ident); + .map(|segment| segment.ident.clone()); x }))?; let ts = quote! { @@ -94,9 +93,8 @@ pub(crate) fn impl_subsystem_sender( support_crate: &TokenStream, outgoing_wrapper: &Ident, subsystem_sender_name: &Ident, - subsystem_ctx_name: &Ident, ) -> TokenStream { - let ts = quote! { + let mut ts = quote! { /// Connector to send messages towards all subsystems, /// while tracking the which signals where already received. #[derive(Debug, Clone)] @@ -193,7 +191,6 @@ pub(crate) fn impl_subsystem_context( subsystem_ctx_name: &Ident, ) -> TokenStream { let signal = &info.extern_signal_ty; - let message_wrapper = &info.message_wrapper; let error_ty = &info.extern_error_ty; let support_crate = info.support_crate_name(); diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index b6bf19e8f401..d87a391c6890 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -26,8 +26,7 @@ mod impl_dispatch; mod impl_message_wrapper; mod impl_overseer; mod impl_subsystem; -mod parse_attr; -mod parse_struct; +mod parse; use impl_builder::*; use impl_channels_out::*; @@ -35,8 +34,7 @@ use impl_dispatch::*; use impl_message_wrapper::*; use impl_overseer::*; use impl_subsystem::*; -use parse_attr::*; -use parse_struct::*; +use parse::*; #[cfg(test)] mod tests; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs new file mode 100644 index 000000000000..a99232458c46 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs @@ -0,0 +1,24 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +mod parse_attr; +mod parse_struct; + +#[cfg(test)] +mod tests; + +pub(crate) use parse_attr::*; +pub(crate) use parse_struct::*; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_attr.rs similarity index 100% rename from node/overseer/overseer-gen/proc-macro/src/parse_attr.rs rename to node/overseer/overseer-gen/proc-macro/src/parse/parse_attr.rs diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs similarity index 86% rename from node/overseer/overseer-gen/proc-macro/src/parse_struct.rs rename to node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs index 51532e3dbb53..132f74620069 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs @@ -17,7 +17,7 @@ use proc_macro2::{Span, TokenStream}; use std::collections::{hash_map::RandomState, HashMap, HashSet}; use syn::{ - parse::{Parse, ParseStream}, + parse::{Parse, ParseStream, ParseBuffer}, punctuated::Punctuated, spanned::Spanned, token::Bracket, @@ -36,7 +36,7 @@ mod kw { } #[derive(Clone, Debug)] -enum SubSysAttrItem { +pub(crate) enum SubSysAttrItem { /// The subsystem is still a work in progress /// and should not be communicated with. Wip(kw::wip), @@ -61,12 +61,10 @@ impl Parse for SubSysAttrItem { Self::Blocking(input.parse::()?) } else if lookahead.peek(kw::no_dispatch) { Self::NoDispatch(input.parse::()?) - } else if lookahead.peek(kw::consumes) { - Self::Consumes(input.parse::()?) } else if lookahead.peek(kw::sends) { Self::Sends(input.parse::()?) } else { - return Err(lookahead.error()) + Self::Consumes(input.parse::()?) }) } } @@ -156,8 +154,11 @@ macro_rules! extract_variant { #[derive(Debug, Clone)] pub(crate) struct Sends { + #[allow(dead_code)] pub(crate) keyword_sends: kw::sends, + #[allow(dead_code)] pub(crate) colon: Token![:], + #[allow(dead_code)] pub(crate) bracket: Bracket, pub(crate) sends: Punctuated, } @@ -169,32 +170,41 @@ impl Parse for Sends { keyword_sends: input.parse()?, colon: input.parse()?, bracket: syn::bracketed!(content in input), - sends: Punctuated::parse_terminated(&mut content)?, + sends: dbg!(Punctuated::parse_terminated(dbg!(&content))?), }) } } #[derive(Debug, Clone)] pub(crate) struct Consumes { - pub(crate) keyword_consumes: kw::consumes, - pub(crate) colon: Token![:], - pub(crate) bracket: Bracket, - pub(crate) consumes: Punctuated, + #[allow(dead_code)] + pub(crate) keyword_consumes: Option, + #[allow(dead_code)] + pub(crate) colon: Option, + pub(crate) consumes: Path, } impl Parse for Consumes { fn parse(input: syn::parse::ParseStream) -> Result { - let content; - Ok(Self { - keyword_consumes: input.parse()?, - colon: input.parse()?, - bracket: syn::bracketed!(content in input), - consumes: Punctuated::parse_terminated(&mut content)?, + let lookahead = input.lookahead1(); + Ok(if lookahead.peek(kw::consumes) { + Self { + keyword_consumes: Some(input.parse()?), + colon: input.parse()?, + consumes: input.parse()?, + } + } else { + Self { + keyword_consumes: None, + colon: None, + consumes: input.parse()?, + } }) } } -pub(crate) struct SubSystemTags { +#[derive(Debug, Clone)] +pub(crate) struct SubSystemAttributes { #[allow(dead_code)] pub(crate) attrs: Vec, #[allow(dead_code)] @@ -210,22 +220,14 @@ pub(crate) struct SubSystemTags { pub(crate) sends: Option, } -impl Parse for SubSystemTags { +impl Parse for SubSystemAttributes { fn parse(input: syn::parse::ParseStream) -> Result { let span = input.span(); - let attrs = Attribute::parse_outer(input)?; - - let input = input; - let content; - let _ = syn::parenthesized!(content in input); - - let mut items = Punctuated::new(); - while let Ok(tag) = content.call(SubSysAttrItem::parse) { - items.push_value(tag); - items.push_punct(content.call(::parse)?); - } + let attrs = dbg!(Attribute::parse_outer(input))?; - assert!(items.empty_or_trailing(), "Always followed by the message type to consume. qed"); + // FIXME assert that it's `subsystem(..)`. + // attrs[0].path + let items = attrs[0].parse_args_with(Punctuated::::parse_terminated)?; let mut unique = HashMap::< std::mem::Discriminant, @@ -378,7 +380,7 @@ impl OverseerInfo { } pub(crate) fn any_message(&self) -> Vec { - self.subsystems.iter().map(|ssf| ssf.message_to_consume).collect::>() + self.subsystems.iter().map(|ssf| ssf.message_to_consume.clone()).collect::>() } pub(crate) fn channel_names_without_wip(&self, suffix: &'static str) -> Vec { @@ -421,7 +423,7 @@ impl OverseerGuts { // for the builder pattern besides other places. let mut unique_subsystem_idents = HashSet::::new(); for Field { attrs, vis, ident, ty, .. } in fields.named.into_iter() { - let mut consumes = + let mut subsystem_attr = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { let span = attr.path.span(); attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { @@ -432,23 +434,20 @@ impl OverseerGuts { let ident = ident.ok_or_else(|| Error::new(ty.span(), "Missing identifier for member. BUG"))?; - if let Some((attr_tokens, span)) = consumes.next() { - if let Some((_attr_tokens2, span2)) = consumes.next() { + // a `#[subsystem(..)]` annotation exists + if let Some((attr_tokens, span)) = subsystem_attr.next() { + if let Some((_attr_tokens2, span2)) = subsystem_attr.next() { return Err({ let mut err = Error::new(span, "The first subsystem annotation is at"); err.combine(Error::new(span2, "but another here for the same field.")); err }) } - let mut consumes_paths = Vec::with_capacity(attrs.len()); - let attr_tokens = attr_tokens.clone(); - let variant: SubSystemTags = syn::parse2(attr_tokens.clone())?; - consumes_paths.push(variant.consumes); - let mut sends_paths = Vec::with_capacity(attrs.len()); + let span = attr_tokens.span(); + let attr_tokens = attr_tokens.clone(); - let variant: SubSystemTags = syn::parse2(attr_tokens.clone())?; - sends_paths.push(variant.consumes); + let subsystem_attrs: SubSystemAttributes = syn::parse2(attr_tokens.clone())?; let field_ty = try_type_to_path(ty, span)?; let generic = field_ty @@ -460,24 +459,47 @@ impl OverseerGuts { ) })? .clone(); + // check for unique subsystem name, otherwise we'd create invalid code: if let Some(previous) = unique_subsystem_idents.get(&generic) { let mut e = Error::new( generic.span(), - format!("Duplicate subsystem names `{}`", generic), + "Duplicate subsystem names", ); e.combine(Error::new(previous.span(), "previously defined here.")); return Err(e) } unique_subsystem_idents.insert(generic.clone()); + + let SubSystemAttributes { no_dispatch, wip, blocking, consumes, sends, .. } + = subsystem_attrs; + + // messages to be sent + let sends = if let Some(sends) = sends { + Vec::from_iter(sends.sends.iter().cloned()) + } else { + vec![] + }; + // messages deemed for consumption + let consumes = if let Some(consumes) = consumes { + consumes.consumes + } else { + return Err(Error::new( + span, + "Must provide exactly one consuming message type", + )); + }; + + // TODO move the send and consumes check here + subsystems.push(SubSysField { name: ident, generic, - message_to_consume: consumes_paths[0].clone(), - messages_to_send: send_paths, - no_dispatch: variant.no_dispatch, - wip: variant.wip, - blocking: variant.blocking, + message_to_consume: consumes, + messages_to_send: sends, + no_dispatch, + wip, + blocking, }); } else { let field_ty = try_type_to_path(ty, ident.span())?; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs new file mode 100644 index 000000000000..e35c9db24506 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs @@ -0,0 +1,303 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; +use assert_matches::assert_matches; +use quote::quote; +use syn::parse_quote; +use syn::parse::Parse; +use crate::{SubSystemAttributes, SubSysAttrItem}; + +mod attr { + use super::*; + + + #[test] + fn attr_full_works() { + let attr: AttrArgs = parse_quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, + error=OverseerError, + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity, + signal_channel_capacity, + .. + } => { + assert_eq!(message_channel_capacity, 222); + assert_eq!(signal_channel_capacity, 111); + }); + } + + #[test] + fn attr_partial_works() { + let attr: AttrArgs = parse_quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, + error=OverseerError, + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity: _, + signal_channel_capacity: _, + .. + } => { + }); + } + +} + + + +mod strukt { + +use super::*; + +#[test] +fn parse_subsystem_attr_item_works_00_wip() { + assert_matches!( + syn::parse2::(quote! { + wip + }), Ok(SubSysAttrItem::Wip(_)) => { + }); +} + + +#[test] +fn parse_subsystem_attr_item_works_01_no_dispatch() { + assert_matches!( + syn::parse2::(quote! { + no_dispatch + }), Ok(SubSysAttrItem::NoDispatch(_)) => { + }); +} + +#[test] +fn parse_subsystem_attr_item_works_02_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [A, B, C] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 3); + }); +} + +#[test] +fn parse_subsystem_attr_item_works_03_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [A] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 1); + }); +} + +#[test] +fn parse_subsystem_attr_item_works_04_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [A,] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 1); + }); +} + +#[test] +fn parse_subsystem_attr_item_works_05_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 0); + }); +} + + +#[test] +fn parse_subsystem_attr_item_works_06_consumes() { + assert_matches!( + syn::parse2::(quote! { + consumes: Foo + }), Ok(SubSysAttrItem::Consumes(_consumes)) => { + }); +} + + + +#[test] +fn parse_subsystem_attr_item_works_07_consumes() { + assert_matches!( + syn::parse2::(quote! { + Foo + }), Ok(SubSysAttrItem::Consumes(_consumes)) => { + }); +} + + + +#[test] +fn parse_subsystem_attributes_works_00() { + syn::parse2::(quote! { + #[subsystem(wip, no_dispatch, blocking, consumes: Foo, sends: [])] + }).unwrap(); +} + +#[test] +fn parse_subsystem_attributes_works_01() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(blocking, Foo, sends: [])] + }), Ok(_) => { + }); +} + +#[test] +fn parse_subsystem_attributes_works_02() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(consumes: Foo, sends: [Bar])] + }), Ok(_) => { + }); +} + +#[test] +fn parse_subsystem_attributes_works_03() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(blocking, consumes: Foo, sends: [Bar])] + }), Ok(_) => { + }); +} + +#[test] +fn parse_subsystem_attributes_works_04() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(wip, consumes: Foo, sends: [Bar])] + }), Ok(_) => { + }); +} + +#[test] +fn parse_subsystem_attributes_works_05() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(consumes: Foo)] + }), Ok(_) => { + }); +} + +#[test] +fn parse_subsystem_attributes_works_06() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(sends: [Foo], consumes: Bar)] + }), Ok(_) => { + }); + +} + +#[test] +fn parse_subsystem_attributes_works_07() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(sends: [Foo], Bar, Y)] + }), Err(e) => { + dbg!(e) + }); +} + +#[test] +fn parse_subsystem_attributes_works_08() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(sends: [Foo], consumes: Bar)] + }), Ok(_) => { + }); +} + +#[test] +fn parse_subsystem_attributes_works_09() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(no_dispatch, sends: [])] + }), Err(e) => { + // must either consume smth or sends smth, neither is NOK + dbg!(e) + }); +} + +#[test] +fn parse_subsystem_attributes_works_10() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem()] + }), Err(e) => { + dbg!(e) + }); +} + +#[test] +fn parse_subsystem_attributes_works_11() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem] + }), Err(e) => { + dbg!(e) + }); +} + + +#[test] +fn parse_subsystem_attributes_works_12() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(Foo)] + }), Ok(_) => { + }); +} + +#[test] +fn struct_parse_full() { + let item: OverseerGuts = parse_quote! { + pub struct Ooooh where X: Secrit { + #[subsystem(no_dispatch, consumes: Foo, sends: [])] + sub0: FooSubsystem, + + #[subsystem(blocking, consumes: Bar, sends: [])] + yyy: BaersBuyBilliardBalls, + + #[subsystem(no_dispatch, blocking, consumes: Twain, sends: [])] + fff: Beeeeep, + + #[subsystem(consumes: Rope)] + mc: MountainCave, + + metrics: Metrics, + } + }; + let _ = dbg!(item); +} + +#[test] +fn struct_parse_basic() { + let item: OverseerGuts = parse_quote! { + pub struct Ooooh { + #[subsystem(consumes: Foo, sends: [])] + sub0: FooSubsystem, + } + }; + let _ = dbg!(item); +} + +} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 3184e88c2ca5..b133a28dd850 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -398,10 +398,10 @@ pub trait SubsystemContext: Send + 'static { /// Send a direct message to some other `Subsystem`, routed based on message type. async fn send_message(&mut self, msg: X) where - Self::AllMessages: From, + Self::OutgoingMessages: From, X: Send, { - self.sender().send_message(::from(msg)).await + self.sender().send_message(::from(msg)).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. @@ -413,7 +413,7 @@ pub trait SubsystemContext: Send + 'static { X: Send, { self.sender() - .send_messages(msgs.into_iter().map(|x| ::from(x))) + .send_messages(msgs.into_iter().map(|x| ::from(x))) .await } @@ -423,7 +423,7 @@ pub trait SubsystemContext: Send + 'static { Self::OutgoingMessages: From, X: Send, { - self.sender().send_unbounded_message(Self::AllMessages::from(msg)) + self.sender().send_unbounded_message(Self::OutgoingMessages::from(msg)) } /// Obtain the sender. diff --git a/node/overseer/src/dummy.rs b/node/overseer/src/dummy.rs index eb6b9099a8bb..1b4dfc82a5d7 100644 --- a/node/overseer/src/dummy.rs +++ b/node/overseer/src/dummy.rs @@ -33,7 +33,7 @@ where Context: SubsystemContext< Signal = OverseerSignal, Error = SubsystemError, - AllMessages = AllMessages, + OutgoingMessages = DummySubsystemOutgoingMessages, >, { fn start(self, mut ctx: Context) -> SpawnedSubsystem { From c23c59bdeacd80ce0ee415cffccb32a3a8a595b3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 14 Apr 2022 19:59:14 +0200 Subject: [PATCH 003/139] fmt --- .../proc-macro/src/parse/parse_struct.rs | 31 +- .../proc-macro/src/parse/tests.rs | 541 +++++++++--------- 2 files changed, 276 insertions(+), 296 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs index 132f74620069..c441ada8007a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs @@ -17,7 +17,7 @@ use proc_macro2::{Span, TokenStream}; use std::collections::{hash_map::RandomState, HashMap, HashSet}; use syn::{ - parse::{Parse, ParseStream, ParseBuffer}, + parse::{Parse, ParseBuffer, ParseStream}, punctuated::Punctuated, spanned::Spanned, token::Bracket, @@ -194,11 +194,7 @@ impl Parse for Consumes { consumes: input.parse()?, } } else { - Self { - keyword_consumes: None, - colon: None, - consumes: input.parse()?, - } + Self { keyword_consumes: None, colon: None, consumes: input.parse()? } }) } } @@ -227,7 +223,8 @@ impl Parse for SubSystemAttributes { // FIXME assert that it's `subsystem(..)`. // attrs[0].path - let items = attrs[0].parse_args_with(Punctuated::::parse_terminated)?; + let items = + attrs[0].parse_args_with(Punctuated::::parse_terminated)?; let mut unique = HashMap::< std::mem::Discriminant, @@ -380,7 +377,10 @@ impl OverseerInfo { } pub(crate) fn any_message(&self) -> Vec { - self.subsystems.iter().map(|ssf| ssf.message_to_consume.clone()).collect::>() + self.subsystems + .iter() + .map(|ssf| ssf.message_to_consume.clone()) + .collect::>() } pub(crate) fn channel_names_without_wip(&self, suffix: &'static str) -> Vec { @@ -461,18 +461,14 @@ impl OverseerGuts { .clone(); // check for unique subsystem name, otherwise we'd create invalid code: if let Some(previous) = unique_subsystem_idents.get(&generic) { - let mut e = Error::new( - generic.span(), - "Duplicate subsystem names", - ); + let mut e = Error::new(generic.span(), "Duplicate subsystem names"); e.combine(Error::new(previous.span(), "previously defined here.")); return Err(e) } unique_subsystem_idents.insert(generic.clone()); - - let SubSystemAttributes { no_dispatch, wip, blocking, consumes, sends, .. } - = subsystem_attrs; + let SubSystemAttributes { no_dispatch, wip, blocking, consumes, sends, .. } = + subsystem_attrs; // messages to be sent let sends = if let Some(sends) = sends { @@ -484,10 +480,7 @@ impl OverseerGuts { let consumes = if let Some(consumes) = consumes { consumes.consumes } else { - return Err(Error::new( - span, - "Must provide exactly one consuming message type", - )); + return Err(Error::new(span, "Must provide exactly one consuming message type")) }; // TODO move the send and consumes check here diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs index e35c9db24506..a3456116ad6d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs @@ -15,289 +15,276 @@ // along with Polkadot. If not, see . use super::*; +use crate::{SubSysAttrItem, SubSystemAttributes}; use assert_matches::assert_matches; use quote::quote; -use syn::parse_quote; -use syn::parse::Parse; -use crate::{SubSystemAttributes, SubSysAttrItem}; +use syn::{parse::Parse, parse_quote}; mod attr { - use super::*; - - - #[test] - fn attr_full_works() { - let attr: AttrArgs = parse_quote! { - gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, - error=OverseerError, - }; - assert_matches!(attr, AttrArgs { - message_channel_capacity, - signal_channel_capacity, - .. - } => { - assert_eq!(message_channel_capacity, 222); - assert_eq!(signal_channel_capacity, 111); - }); - } - - #[test] - fn attr_partial_works() { - let attr: AttrArgs = parse_quote! { - gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, - error=OverseerError, - }; - assert_matches!(attr, AttrArgs { - message_channel_capacity: _, - signal_channel_capacity: _, - .. - } => { - }); - } - + use super::*; + + #[test] + fn attr_full_works() { + let attr: AttrArgs = parse_quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, + error=OverseerError, + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity, + signal_channel_capacity, + .. + } => { + assert_eq!(message_channel_capacity, 222); + assert_eq!(signal_channel_capacity, 111); + }); + } + + #[test] + fn attr_partial_works() { + let attr: AttrArgs = parse_quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, + error=OverseerError, + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity: _, + signal_channel_capacity: _, + .. + } => { + }); + } } - - mod strukt { -use super::*; - -#[test] -fn parse_subsystem_attr_item_works_00_wip() { - assert_matches!( - syn::parse2::(quote! { - wip - }), Ok(SubSysAttrItem::Wip(_)) => { - }); -} - - -#[test] -fn parse_subsystem_attr_item_works_01_no_dispatch() { - assert_matches!( - syn::parse2::(quote! { - no_dispatch - }), Ok(SubSysAttrItem::NoDispatch(_)) => { - }); -} - -#[test] -fn parse_subsystem_attr_item_works_02_sends() { - assert_matches!( - syn::parse2::(quote! { - sends: [A, B, C] - }), Ok(SubSysAttrItem::Sends(sends)) => { - assert_eq!(sends.sends.len(), 3); - }); -} - -#[test] -fn parse_subsystem_attr_item_works_03_sends() { - assert_matches!( - syn::parse2::(quote! { - sends: [A] - }), Ok(SubSysAttrItem::Sends(sends)) => { - assert_eq!(sends.sends.len(), 1); - }); -} - -#[test] -fn parse_subsystem_attr_item_works_04_sends() { - assert_matches!( - syn::parse2::(quote! { - sends: [A,] - }), Ok(SubSysAttrItem::Sends(sends)) => { - assert_eq!(sends.sends.len(), 1); - }); -} - -#[test] -fn parse_subsystem_attr_item_works_05_sends() { - assert_matches!( - syn::parse2::(quote! { - sends: [] - }), Ok(SubSysAttrItem::Sends(sends)) => { - assert_eq!(sends.sends.len(), 0); - }); -} - - -#[test] -fn parse_subsystem_attr_item_works_06_consumes() { - assert_matches!( - syn::parse2::(quote! { - consumes: Foo - }), Ok(SubSysAttrItem::Consumes(_consumes)) => { - }); -} - - - -#[test] -fn parse_subsystem_attr_item_works_07_consumes() { - assert_matches!( - syn::parse2::(quote! { - Foo - }), Ok(SubSysAttrItem::Consumes(_consumes)) => { - }); -} - - - -#[test] -fn parse_subsystem_attributes_works_00() { - syn::parse2::(quote! { - #[subsystem(wip, no_dispatch, blocking, consumes: Foo, sends: [])] - }).unwrap(); -} - -#[test] -fn parse_subsystem_attributes_works_01() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(blocking, Foo, sends: [])] - }), Ok(_) => { - }); -} - -#[test] -fn parse_subsystem_attributes_works_02() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(consumes: Foo, sends: [Bar])] - }), Ok(_) => { - }); -} - -#[test] -fn parse_subsystem_attributes_works_03() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(blocking, consumes: Foo, sends: [Bar])] - }), Ok(_) => { - }); -} - -#[test] -fn parse_subsystem_attributes_works_04() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(wip, consumes: Foo, sends: [Bar])] - }), Ok(_) => { - }); -} - -#[test] -fn parse_subsystem_attributes_works_05() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(consumes: Foo)] - }), Ok(_) => { - }); -} - -#[test] -fn parse_subsystem_attributes_works_06() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(sends: [Foo], consumes: Bar)] - }), Ok(_) => { - }); - -} - -#[test] -fn parse_subsystem_attributes_works_07() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(sends: [Foo], Bar, Y)] - }), Err(e) => { - dbg!(e) - }); -} - -#[test] -fn parse_subsystem_attributes_works_08() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(sends: [Foo], consumes: Bar)] - }), Ok(_) => { - }); -} - -#[test] -fn parse_subsystem_attributes_works_09() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(no_dispatch, sends: [])] - }), Err(e) => { - // must either consume smth or sends smth, neither is NOK - dbg!(e) - }); -} - -#[test] -fn parse_subsystem_attributes_works_10() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem()] - }), Err(e) => { - dbg!(e) - }); -} - -#[test] -fn parse_subsystem_attributes_works_11() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem] - }), Err(e) => { - dbg!(e) - }); -} - - -#[test] -fn parse_subsystem_attributes_works_12() { - assert_matches!( - syn::parse2::(quote! { - #[subsystem(Foo)] - }), Ok(_) => { - }); -} - -#[test] -fn struct_parse_full() { - let item: OverseerGuts = parse_quote! { - pub struct Ooooh where X: Secrit { - #[subsystem(no_dispatch, consumes: Foo, sends: [])] - sub0: FooSubsystem, - - #[subsystem(blocking, consumes: Bar, sends: [])] - yyy: BaersBuyBilliardBalls, - - #[subsystem(no_dispatch, blocking, consumes: Twain, sends: [])] - fff: Beeeeep, - - #[subsystem(consumes: Rope)] - mc: MountainCave, - - metrics: Metrics, - } - }; - let _ = dbg!(item); -} - -#[test] -fn struct_parse_basic() { - let item: OverseerGuts = parse_quote! { - pub struct Ooooh { - #[subsystem(consumes: Foo, sends: [])] - sub0: FooSubsystem, - } - }; - let _ = dbg!(item); -} - + use super::*; + + #[test] + fn parse_subsystem_attr_item_works_00_wip() { + assert_matches!( + syn::parse2::(quote! { + wip + }), Ok(SubSysAttrItem::Wip(_)) => { + }); + } + + #[test] + fn parse_subsystem_attr_item_works_01_no_dispatch() { + assert_matches!( + syn::parse2::(quote! { + no_dispatch + }), Ok(SubSysAttrItem::NoDispatch(_)) => { + }); + } + + #[test] + fn parse_subsystem_attr_item_works_02_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [A, B, C] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 3); + }); + } + + #[test] + fn parse_subsystem_attr_item_works_03_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [A] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 1); + }); + } + + #[test] + fn parse_subsystem_attr_item_works_04_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [A,] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 1); + }); + } + + #[test] + fn parse_subsystem_attr_item_works_05_sends() { + assert_matches!( + syn::parse2::(quote! { + sends: [] + }), Ok(SubSysAttrItem::Sends(sends)) => { + assert_eq!(sends.sends.len(), 0); + }); + } + + #[test] + fn parse_subsystem_attr_item_works_06_consumes() { + assert_matches!( + syn::parse2::(quote! { + consumes: Foo + }), Ok(SubSysAttrItem::Consumes(_consumes)) => { + }); + } + + #[test] + fn parse_subsystem_attr_item_works_07_consumes() { + assert_matches!( + syn::parse2::(quote! { + Foo + }), Ok(SubSysAttrItem::Consumes(_consumes)) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_00() { + syn::parse2::(quote! { + #[subsystem(wip, no_dispatch, blocking, consumes: Foo, sends: [])] + }) + .unwrap(); + } + + #[test] + fn parse_subsystem_attributes_works_01() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(blocking, Foo, sends: [])] + }), Ok(_) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_02() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(consumes: Foo, sends: [Bar])] + }), Ok(_) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_03() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(blocking, consumes: Foo, sends: [Bar])] + }), Ok(_) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_04() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(wip, consumes: Foo, sends: [Bar])] + }), Ok(_) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_05() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(consumes: Foo)] + }), Ok(_) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_06() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(sends: [Foo], consumes: Bar)] + }), Ok(_) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_07() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(sends: [Foo], Bar, Y)] + }), Err(e) => { + dbg!(e) + }); + } + + #[test] + fn parse_subsystem_attributes_works_08() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(sends: [Foo], consumes: Bar)] + }), Ok(_) => { + }); + } + + #[test] + fn parse_subsystem_attributes_works_09() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(no_dispatch, sends: [])] + }), Err(e) => { + // must either consume smth or sends smth, neither is NOK + dbg!(e) + }); + } + + #[test] + fn parse_subsystem_attributes_works_10() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem()] + }), Err(e) => { + dbg!(e) + }); + } + + #[test] + fn parse_subsystem_attributes_works_11() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem] + }), Err(e) => { + dbg!(e) + }); + } + + #[test] + fn parse_subsystem_attributes_works_12() { + assert_matches!( + syn::parse2::(quote! { + #[subsystem(Foo)] + }), Ok(_) => { + }); + } + + #[test] + fn struct_parse_full() { + let item: OverseerGuts = parse_quote! { + pub struct Ooooh where X: Secrit { + #[subsystem(no_dispatch, consumes: Foo, sends: [])] + sub0: FooSubsystem, + + #[subsystem(blocking, consumes: Bar, sends: [])] + yyy: BaersBuyBilliardBalls, + + #[subsystem(no_dispatch, blocking, consumes: Twain, sends: [])] + fff: Beeeeep, + + #[subsystem(consumes: Rope)] + mc: MountainCave, + + metrics: Metrics, + } + }; + let _ = dbg!(item); + } + + #[test] + fn struct_parse_basic() { + let item: OverseerGuts = parse_quote! { + pub struct Ooooh { + #[subsystem(consumes: Foo, sends: [])] + sub0: FooSubsystem, + } + }; + let _ = dbg!(item); + } } From 510bed69b12409fb2db7489a1b412e0dbd5cba4c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 19 Apr 2022 10:49:37 +0200 Subject: [PATCH 004/139] better check if missing send --- node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs index c441ada8007a..9169b8a26a1b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs @@ -244,7 +244,7 @@ impl Parse for SubSystemAttributes { // A subsystem makes no sense if not one of them is provided let sends = extract_variant!(unique, Sends take); let consumes = extract_variant!(unique, Consumes take); - if sends.is_none() && consumes.is_none() { + if sends.as_ref().map(|sends| sends.sends.is_empty()).unwrap_or(true) && consumes.is_none() { return Err(Error::new( span, "Must have at least one of `consumes: [..]` and `sends: [..]`.", From 2a49f94eff9a8e9e40d0ec7fdc6beced110d086f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 19 Apr 2022 12:16:04 +0200 Subject: [PATCH 005/139] fix failing tests --- .../proc-macro/src/parse/parse_struct.rs | 47 ++++++----- .../proc-macro/src/parse/tests.rs | 82 +++++++++++-------- 2 files changed, 74 insertions(+), 55 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs index 9169b8a26a1b..4223999ea660 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs @@ -17,12 +17,13 @@ use proc_macro2::{Span, TokenStream}; use std::collections::{hash_map::RandomState, HashMap, HashSet}; use syn::{ - parse::{Parse, ParseBuffer, ParseStream}, + parenthesized, + parse::{Parse, ParseStream}, punctuated::Punctuated, spanned::Spanned, token::Bracket, - AttrStyle, Attribute, Error, Field, FieldsNamed, GenericParam, Ident, ItemStruct, Path, Result, - Token, Type, Visibility, + AttrStyle, Error, Field, FieldsNamed, GenericParam, Ident, ItemStruct, Path, Result, Token, + Type, Visibility, }; use quote::{quote, ToTokens}; @@ -170,7 +171,7 @@ impl Parse for Sends { keyword_sends: input.parse()?, colon: input.parse()?, bracket: syn::bracketed!(content in input), - sends: dbg!(Punctuated::parse_terminated(dbg!(&content))?), + sends: Punctuated::parse_terminated(&content)?, }) } } @@ -199,10 +200,10 @@ impl Parse for Consumes { } } +/// Parses `(no_dispatch, Foo, sends = [Bar, Baz])` +/// including the `(` and `)`. #[derive(Debug, Clone)] -pub(crate) struct SubSystemAttributes { - #[allow(dead_code)] - pub(crate) attrs: Vec, +pub(crate) struct SubSystemAttrItems { #[allow(dead_code)] pub(crate) no_dispatch: bool, /// The subsystem is in progress, only generate the `Wrapper` variant, but do not forward messages @@ -216,15 +217,14 @@ pub(crate) struct SubSystemAttributes { pub(crate) sends: Option, } -impl Parse for SubSystemAttributes { +impl Parse for SubSystemAttrItems { fn parse(input: syn::parse::ParseStream) -> Result { let span = input.span(); - let attrs = dbg!(Attribute::parse_outer(input))?; - // FIXME assert that it's `subsystem(..)`. - // attrs[0].path - let items = - attrs[0].parse_args_with(Punctuated::::parse_terminated)?; + let content; + let _paren_token = parenthesized!(content in input); + + let items = content.call(Punctuated::::parse_terminated)?; let mut unique = HashMap::< std::mem::Discriminant, @@ -244,7 +244,8 @@ impl Parse for SubSystemAttributes { // A subsystem makes no sense if not one of them is provided let sends = extract_variant!(unique, Sends take); let consumes = extract_variant!(unique, Consumes take); - if sends.as_ref().map(|sends| sends.sends.is_empty()).unwrap_or(true) && consumes.is_none() { + if sends.as_ref().map(|sends| sends.sends.is_empty()).unwrap_or(true) && consumes.is_none() + { return Err(Error::new( span, "Must have at least one of `consumes: [..]` and `sends: [..]`.", @@ -255,7 +256,7 @@ impl Parse for SubSystemAttributes { let blocking = extract_variant!(unique, Blocking; default = false); let wip = extract_variant!(unique, Wip; default = false); - Ok(Self { attrs, no_dispatch, blocking, wip, sends, consumes }) + Ok(Self { no_dispatch, blocking, wip, sends, consumes }) } } @@ -423,6 +424,7 @@ impl OverseerGuts { // for the builder pattern besides other places. let mut unique_subsystem_idents = HashSet::::new(); for Field { attrs, vis, ident, ty, .. } in fields.named.into_iter() { + // collect all subsystem annotations per field let mut subsystem_attr = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { let span = attr.path.span(); @@ -431,8 +433,12 @@ impl OverseerGuts { (attr_tokens, span) }) }); - let ident = - ident.ok_or_else(|| Error::new(ty.span(), "Missing identifier for member. BUG"))?; + let ident = ident.ok_or_else(|| { + Error::new( + ty.span(), + "Missing identifier for field, only named fields are expceted.", + ) + })?; // a `#[subsystem(..)]` annotation exists if let Some((attr_tokens, span)) = subsystem_attr.next() { @@ -447,7 +453,7 @@ impl OverseerGuts { let span = attr_tokens.span(); let attr_tokens = attr_tokens.clone(); - let subsystem_attrs: SubSystemAttributes = syn::parse2(attr_tokens.clone())?; + let subsystem_attrs: SubSystemAttrItems = syn::parse2(attr_tokens.clone())?; let field_ty = try_type_to_path(ty, span)?; let generic = field_ty @@ -467,7 +473,7 @@ impl OverseerGuts { } unique_subsystem_idents.insert(generic.clone()); - let SubSystemAttributes { no_dispatch, wip, blocking, consumes, sends, .. } = + let SubSystemAttrItems { no_dispatch, wip, blocking, consumes, sends, .. } = subsystem_attrs; // messages to be sent @@ -484,7 +490,6 @@ impl OverseerGuts { }; // TODO move the send and consumes check here - subsystems.push(SubSysField { name: ident, generic, @@ -499,7 +504,7 @@ impl OverseerGuts { let generic = field_ty .get_ident() .map(|ident| baggage_generics.contains(ident)) - .unwrap_or_default(); + .unwrap_or(false); baggage.push(BaggageField { field_name: ident, generic, field_ty, vis }); } } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs index a3456116ad6d..7032c2e7f9c9 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs @@ -15,10 +15,10 @@ // along with Polkadot. If not, see . use super::*; -use crate::{SubSysAttrItem, SubSystemAttributes}; +use crate::{SubSysAttrItem, SubSystemAttrItems}; use assert_matches::assert_matches; use quote::quote; -use syn::{parse::Parse, parse_quote}; +use syn::parse_quote; mod attr { use super::*; @@ -136,8 +136,8 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_00() { - syn::parse2::(quote! { - #[subsystem(wip, no_dispatch, blocking, consumes: Foo, sends: [])] + syn::parse2::(quote! { + (wip, no_dispatch, blocking, consumes: Foo, sends: []) }) .unwrap(); } @@ -145,8 +145,8 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_01() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(blocking, Foo, sends: [])] + syn::parse2::(quote! { + (blocking, Foo, sends: []) }), Ok(_) => { }); } @@ -154,8 +154,8 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_02() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(consumes: Foo, sends: [Bar])] + syn::parse2::(quote! { + (consumes: Foo, sends: [Bar]) }), Ok(_) => { }); } @@ -163,8 +163,8 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_03() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(blocking, consumes: Foo, sends: [Bar])] + syn::parse2::(quote! { + (blocking, consumes: Foo, sends: [Bar]) }), Ok(_) => { }); } @@ -172,8 +172,8 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_04() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(wip, consumes: Foo, sends: [Bar])] + syn::parse2::(quote! { + (wip, consumes: Foo, sends: [Bar]) }), Ok(_) => { }); } @@ -181,8 +181,8 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_05() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(consumes: Foo)] + syn::parse2::(quote! { + (consumes: Foo) }), Ok(_) => { }); } @@ -190,17 +190,17 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_06() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(sends: [Foo], consumes: Bar)] + syn::parse2::(quote! { + (sends: [Foo], consumes: Bar) }), Ok(_) => { }); } #[test] - fn parse_subsystem_attributes_works_07() { + fn parse_subsystem_attributes_works_07_duplicate_send() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(sends: [Foo], Bar, Y)] + syn::parse2::(quote! { + (sends: [Foo], Bar, Y) }), Err(e) => { dbg!(e) }); @@ -209,17 +209,17 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_08() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(sends: [Foo], consumes: Bar)] + syn::parse2::(quote! { + (sends: [Foo], consumes: Bar) }), Ok(_) => { }); } #[test] - fn parse_subsystem_attributes_works_09() { + fn parse_subsystem_attributes_works_09_neither_consumes_nor_sends() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(no_dispatch, sends: [])] + syn::parse2::(quote! { + (no_dispatch, sends: []) }), Err(e) => { // must either consume smth or sends smth, neither is NOK dbg!(e) @@ -227,34 +227,48 @@ mod strukt { } #[test] - fn parse_subsystem_attributes_works_10() { + fn parse_subsystem_attributes_works_10_empty_with_braces() { assert_matches!( - syn::parse2::(quote! { - #[subsystem()] + syn::parse2::(quote! { + () }), Err(e) => { dbg!(e) }); } #[test] - fn parse_subsystem_attributes_works_11() { + fn parse_subsystem_attributes_works_11_empty() { assert_matches!( - syn::parse2::(quote! { - #[subsystem] + syn::parse2::(quote! { + }), Err(e) => { dbg!(e) }); } #[test] - fn parse_subsystem_attributes_works_12() { + fn parse_subsystem_attributes_works_12_duplicate_consumes_different_fmt() { assert_matches!( - syn::parse2::(quote! { - #[subsystem(Foo)] - }), Ok(_) => { + syn::parse2::(quote! { + (Foo, consumes = Foo) + }), Err(e) => { + dbg!(e) }); } + #[test] + fn struct_parse_baggage() { + let item: OverseerGuts = parse_quote! { + pub struct Ooooh where X: Secrit { + #[subsystem(no_dispatch, consumes: Foo, sends: [])] + sub0: FooSubsystem, + + metrics: Metrics, + } + }; + let _ = dbg!(item); + } + #[test] fn struct_parse_full() { let item: OverseerGuts = parse_quote! { From d7cab79ea6d71554f3d04a5bc6fe8a08f313250d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 19 Apr 2022 12:55:35 +0200 Subject: [PATCH 006/139] impro --- node/overseer/overseer-gen/examples/dummy.rs | 16 +++++-------- .../proc-macro/src/impl_subsystem.rs | 12 +++++----- .../proc-macro/src/parse/parse_struct.rs | 23 ++++++++++++++----- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index f804f2906167..92d820366bd3 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -8,8 +8,8 @@ use std::collections::HashMap; #[derive(Default)] pub struct AwesomeSubSys; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { - fn start(self, _ctx: XxxSubsystemContext) -> SpawnedSubsystem { +impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { + fn start(self, ctx: SubsystemContext) -> SpawnedSubsystem { self.spawn(async move { ctx.send_message(Plinko).await }); unimplemented!("starting yay!") } @@ -18,8 +18,8 @@ impl ::polkadot_overseer_gen::Subsystem, Yikes> f #[derive(Default)] pub struct GoblinTower; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { - fn start(self, _ctx: XxxSubsystemContext) -> SpawnedSubsystem { +impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { + fn start(self, ctx: SubsystemContext) -> SpawnedSubsystem { self.spawn(async move { ctx.send_message(MsgStrukt).await }); unimplemented!("welcum") } @@ -93,16 +93,12 @@ impl NetworkMsg { #[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] struct Xxx { #[subsystem( - handles: MsgStrukt, + consumes: MsgStrukt, sends: Plinko )] sub0: AwesomeSubSys, - #[subsystem(no_dispatch, blocking, - handles: [Plinko], - sends: [ - MsgStruckt - ]) + #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: MsgStruckt) ] plinkos: GoblinTower, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 843e754b5b40..8fa4e4e36fc7 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -28,7 +28,7 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let support_crate = info.support_crate_name(); for ssf in info.subsystems() { - let subsystem_name = ssf.name.to_string(); + let subsystem_name = ssf.generic.to_string(); let subsystem_sender_name = Ident::new(&(subsystem_name.clone() + "SubsystemSender"), span); let subsystem_ctx_name = Ident::new(&(subsystem_name.clone() + "SubsystemContext"), span); @@ -143,18 +143,18 @@ pub(crate) fn impl_subsystem_sender( // // 1. subsystem specific `*OutgoingMessages`-type // 2. overseer-global-`AllMessages`-type - let wrapped = |wrapper: &Ident| { + let wrapped = |outgoing_wrapper: &Ident| { quote! { /// implementation for wrapping message type... #[#support_crate ::async_trait] - impl SubsystemSender< #wrapper > for #subsystem_sender_name { - async fn send_message(&mut self, msg: #wrapper) { + impl SubsystemSender< #outgoing_wrapper > for #subsystem_sender_name { + async fn send_message(&mut self, msg: #outgoing_wrapper) { self.channels.send_and_log_error(self.signals_received.load(), msg).await; } async fn send_messages(&mut self, msgs: T) where - T: IntoIterator + Send, + T: IntoIterator + Send, T::IntoIter: Send, { // This can definitely be optimized if necessary. @@ -163,7 +163,7 @@ pub(crate) fn impl_subsystem_sender( } } - fn send_unbounded_message(&mut self, msg: #wrapper) { + fn send_unbounded_message(&mut self, msg: #outgoing_wrapper) { self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); } } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs index 4223999ea660..7fd849916cb5 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs @@ -160,18 +160,29 @@ pub(crate) struct Sends { #[allow(dead_code)] pub(crate) colon: Token![:], #[allow(dead_code)] - pub(crate) bracket: Bracket, + pub(crate) bracket: Option, pub(crate) sends: Punctuated, } impl Parse for Sends { fn parse(input: syn::parse::ParseStream) -> Result { let content; - Ok(Self { - keyword_sends: input.parse()?, - colon: input.parse()?, - bracket: syn::bracketed!(content in input), - sends: Punctuated::parse_terminated(&content)?, + let keyword_sends = input.parse()?; + let colon = input.parse()?; + let (bracket, sends) = if !input.peek(syn::token::Bracket) { + let mut sends = Punctuated::new(); + sends.push_value(input.parse::()?); + (None, sends) + } else { + let bracket = Some(syn::bracketed!(content in input)); + let sends = Punctuated::parse_terminated(&content)?; + (bracket, sends) + }; + Ok(Self { + keyword_sends, + colon, + bracket, + sends, }) } } From 63cfb6223e198a419944ae49075d6de45428679a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 19 Apr 2022 18:53:51 +0200 Subject: [PATCH 007/139] intermediate commit, use petgraph for convenience and the dot graph --- Cargo.lock | 1 + node/overseer/overseer-gen/examples/dummy.rs | 24 +- .../overseer-gen/proc-macro/Cargo.toml | 3 +- .../proc-macro/src/impl_builder.rs | 18 +- .../proc-macro/src/impl_subsystem.rs | 270 +++++++++++++----- .../proc-macro/src/parse/parse_struct.rs | 7 +- node/overseer/overseer-gen/src/lib.rs | 20 +- 7 files changed, 242 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2157d421d8a..38675f9ceec9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7028,6 +7028,7 @@ version = "0.9.19" dependencies = [ "assert_matches", "expander 0.0.6", + "petgraph", "proc-macro-crate", "proc-macro2", "quote", diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 92d820366bd3..1ec429161d00 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -8,9 +8,9 @@ use std::collections::HashMap; #[derive(Default)] pub struct AwesomeSubSys; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { - fn start(self, ctx: SubsystemContext) -> SpawnedSubsystem { - self.spawn(async move { ctx.send_message(Plinko).await }); +impl ::polkadot_overseer_gen::Subsystem for AwesomeSubSys { + fn start(self, ctx: AwesomeSubSysContext) -> SpawnedSubsystem { + ctx.spawn("awesome", Box::pin(async move { ctx.send_message(Plinko).await })); unimplemented!("starting yay!") } } @@ -18,9 +18,9 @@ impl ::polkadot_overseer_gen::Subsystem, Yikes> for #[derive(Default)] pub struct GoblinTower; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { - fn start(self, ctx: SubsystemContext) -> SpawnedSubsystem { - self.spawn(async move { ctx.send_message(MsgStrukt).await }); +impl ::polkadot_overseer_gen::Subsystem for GoblinTower { + fn start(self, ctx: GoblinTowerContext) -> SpawnedSubsystem { + ctx.spawn("awesome", Box::pin(async move { ctx.send_message(MsgStrukt(0u8)).await })); unimplemented!("welcum") } } @@ -91,15 +91,11 @@ impl NetworkMsg { } #[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] -struct Xxx { - #[subsystem( - consumes: MsgStrukt, - sends: Plinko - )] +struct Yyy { + #[subsystem(consumes: MsgStrukt, sends: Plinko)] sub0: AwesomeSubSys, - #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: MsgStruckt) - ] + #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: MsgStrukt)] plinkos: GoblinTower, i_like_pi: f64, @@ -134,7 +130,7 @@ impl SpawnNamed for DummySpawner { struct DummyCtx; fn main() { - let (overseer, _handle): (Xxx<_, f64>, _) = Xxx::builder() + let (overseer, _handle): (Yyy<_, f64>, _) = Yyy::builder() .sub0(AwesomeSubSys::default()) .plinkos(GoblinTower::default()) .i_like_pi(::std::f64::consts::PI) diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml index 436384ca0acb..289542926338 100644 --- a/node/overseer/overseer-gen/proc-macro/Cargo.toml +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -17,12 +17,13 @@ quote = "1.0.18" proc-macro2 = "1.0.37" proc-macro-crate = "1.1.3" expander = { version = "0.0.6", default-features = false } +petgraph = "0.6.0" [dev-dependencies] assert_matches = "1.5.0" [features] -default = [] +default = ["expand"] # write the expanded version to a `overlord-expansion.[a-f0-9]{10}.rs` # in the `OUT_DIR` as defined by `cargo` for the `expander` crate. expand = [] diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 4e9b96830e3b..6c33a385efd0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -35,10 +35,11 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { let builder = format_ident!("{}Builder", overseer_name); let handle = format_ident!("{}Handle", overseer_name); let connector = format_ident!("{}Connector", overseer_name); - let subsystem_ctx_name = format_ident!("{}SubsystemContext", overseer_name); let subsystem_name = &info.subsystem_names_without_wip(); let subsystem_generics = &info.subsystem_generic_types(); + let subsystem_ctx_names = + &Vec::from_iter(subsystem_generics.iter().map(|name| format_ident!("{}Context", name))); let consumes = &info.consumes_without_wip(); let channel_name = &info.channel_names_without_wip(""); @@ -103,7 +104,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { info.subsystems().iter().filter(|ssf| !ssf.wip).enumerate().map(|(idx, ssf)| { let field_name = &ssf.name; let field_type = &ssf.generic; - let subsystem_consumes = &ssf.message_to_consume; + let subsystem_ctx_name = format_ident!("{}Context", &field_type); + // Remove state generic for the item to be replaced. It sufficient to know `field_type` for // that since we always move from `Init<#field_type>` to `Init`. let impl_subsystem_state_generics = recollect_without_idx(&subsystem_passthrough_state_generics[..], idx); @@ -135,7 +137,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { impl #builder where - #field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>, + #field_type : Subsystem<#subsystem_ctx_name, #error_ty>, { /// Specify the subsystem in the builder directly pub fn #field_name (self, var: #field_type ) -> @@ -186,7 +188,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { impl #builder where - #field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>, + #field_type : Subsystem<#subsystem_ctx_name, #error_ty>, { /// Replace a subsystem by another implementation for the /// consumable message type. @@ -195,7 +197,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { where #field_type: 'static, F: 'static + FnOnce(#field_type) -> NEW, - NEW: #support_crate ::Subsystem<#subsystem_ctx_name< #subsystem_consumes >, #error_ty>, + NEW: #support_crate ::Subsystem<#subsystem_ctx_name, #error_ty>, { let replacement: Init = match self.#field_name { Init::Fn(fx) => @@ -339,7 +341,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { #builder #(, Missing<#field_type> )* > where #( - #subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>, + #subsystem_generics : Subsystem<#subsystem_ctx_names, #error_ty>, )* { #builder :: new() @@ -500,7 +502,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { where #spawner_where_clause, #( - #subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>, + #subsystem_generics : Subsystem<#subsystem_ctx_names, #error_ty>, )* { /// Complete the construction and create the overseer type. @@ -583,7 +585,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { // Convert owned `snake case` string to a `kebab case` static str. let subsystem_static_str = Box::leak(subsystem_string.replace("_", "-").into_boxed_str()); - let ctx = #subsystem_ctx_name::< #consumes >::new( + let ctx = #subsystem_ctx_names::new( signal_rx, message_rx, channels_out.clone(), diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 8fa4e4e36fc7..0556376b782d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -18,8 +18,39 @@ use proc_macro2::TokenStream; use quote::quote; use syn::{Ident, Path, Result}; +use petgraph::{ + dot::{self, Dot}, + graph::NodeIndex, + visit::EdgeRef, + Direction, +}; +use std::collections::HashMap; + use super::*; +/// Render a graphviz (aka dot graph) to a file. +fn graphviz( + graph: &petgraph::Graph, + dest: &mut impl std::io::Write, +) -> std::io::Result<()> { + let config = &[dot::Config::EdgeNoLabel, dot::Config::NodeNoLabel][..]; + let dot = Dot::with_attr_getters( + graph, + config, + &|_graph, edge| -> String { + format!( + r#"label="{}""#, + edge.weight().get_ident().expect("Must have a trailing identifier. qed") + ) + }, + &|_graph, (node_index, subsystem_name)| -> String { + format!(r#"label="{}""#, subsystem_name,) + }, + ); + dest.write_all(format!("{:?}", &dot).as_bytes())?; + Ok(()) +} + pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let mut ts = TokenStream::new(); @@ -27,23 +58,104 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let all_messages_wrapper = &info.message_wrapper; let support_crate = info.support_crate_name(); + // create a directed graph with all the subsystems as nodes and the messages as edges + // key is always the message path, values are node indices in the graph and the subsystem generic identifier + // store the message path and the source sender, both in the graph as well as identifier + let mut outgoing_lut = HashMap::<&Path, Vec<(Ident, NodeIndex)>>::with_capacity(128); + // same for consuming the incoming messages + let mut consuming_lut = HashMap::<&Path, (Ident, NodeIndex)>::with_capacity(128); + + // Ident = Node = subsystem generic names + // Path = Edge = messages + let mut graph = petgraph::Graph::::new(); + + // prepare the full index of outgoing and source subsystems + for ssf in info.subsystems() { + let node_index = graph.add_node(ssf.generic.clone()); + for outgoing in ssf.messages_to_send.iter() { + outgoing_lut + .entry(outgoing) + .or_default() + .push((ssf.generic.clone(), node_index)); + } + consuming_lut.insert(&ssf.message_to_consume, (ssf.generic.clone(), node_index)); + } + + for (message_ty, (consuming_subsystem_ident, consuming_node_index)) in consuming_lut.iter() { + // match the outgoing ones that were registered above with the consumed message + if let Some(origin_subsystems) = outgoing_lut.get(message_ty) { + for (origin_subsystem, sending_node_index) in origin_subsystems.iter() { + graph.add_edge(*sending_node_index, *consuming_node_index, (*message_ty).clone()); + } + } + } + + // all outgoing edges are now usable to deriver everything we need + for node_index in graph.node_indices() { + let subsystem_name = graph[node_index].to_string(); + let outgoing_wrapper = Ident::new(&(subsystem_name + "OutgoingMessages"), span); + + // cannot be a hashmap, duplicate keys and sorting required + let outgoing_to_consumer = graph + .edges_directed(node_index, Direction::Outgoing) + .map(|edge| { + let message_ty = edge.weight(); + let subsystem_generic_consumer = graph[edge.target()].clone(); + Ok((to_variant(message_ty, span.clone())?, subsystem_generic_consumer)) + }) + .collect::>>()?; + + let outgoing_variant = outgoing_to_consumer.iter().map(|x| x.0.clone()).collect::>(); + let subsystem_generic = outgoing_to_consumer.into_iter().map(|x| x.1).collect::>(); + + ts.extend(quote! { + impl ::std::convert::From< #outgoing_wrapper > for #all_messages_wrapper { + fn from(message: #outgoing_wrapper) -> Self { + match message { + #( + #outgoing_wrapper :: #outgoing_variant ( msg ) => #all_messages_wrapper :: #subsystem_generic ( msg ), + )* + } + } + } + }) + } + + if true { + let mut f = std::fs::OpenOptions::new() + .truncate(true) + .create(true) + .write(true) + .open("/tmp/foo.dot") + .expect("Opening that file works. qed"); + graphviz(&graph, &mut f).expect("Write of dot graph should work. qed"); + } + for ssf in info.subsystems() { let subsystem_name = ssf.generic.to_string(); - let subsystem_sender_name = Ident::new(&(subsystem_name.clone() + "SubsystemSender"), span); - let subsystem_ctx_name = Ident::new(&(subsystem_name.clone() + "SubsystemContext"), span); + let subsystem_sender_name = Ident::new(&(subsystem_name.clone() + "Sender"), span); + let subsystem_ctx_name = Ident::new(&(subsystem_name.clone() + "Context"), span); let outgoing_wrapper = Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span); + let consumes = ssf + .message_to_consume + .get_ident() + .cloned() + .unwrap_or_else(|| Ident::new("()", Span::call_site())); ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?); + ts.extend(impl_subsystem_sender( ssf, &all_messages_wrapper, support_crate, &outgoing_wrapper, + &all_messages_wrapper, &subsystem_sender_name, )); ts.extend(impl_subsystem_context( info, + &consumes, &outgoing_wrapper, &subsystem_sender_name, &subsystem_ctx_name, @@ -52,19 +164,64 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { Ok(ts) } +fn to_variant(path: &Path, span: Span) -> Result { + let ident = path + .segments + .last() + .ok_or_else(|| syn::Error::new(span, "Path is empty, but it must end with an identifier")) + .map(|segment| segment.ident.clone())?; + Ok(ident) +} + +fn to_variants(message_types: &[Path], span: Span) -> Result> { + let variants: Vec<_> = + Result::from_iter(message_types.into_iter().map(|path| to_variant(path, span.clone())))?; + Ok(variants) +} + /// Generates the wrapper type enum, no bells or whistles. pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Result { + // The message types are path based, each of them must finish with a type + // and as such we do this upfront. + let variants = to_variants(message_types, wrapper.span())?; + + let ts = quote! { + #[allow(missing_docs)] + #[derive(Clone)] + pub enum #wrapper { + #( + #variants ( #message_types ), + )* + } + + #( + impl ::std::convert::From< #message_types > for #wrapper { + fn from(message: #message_types) -> Self { + #wrapper :: #variants ( message ) + } + } + )* + }; + Ok(ts) +} + +/// Generates the wrapper type enum, no bells or whistles. +pub(crate) fn impl_wrapper_to_wrapper_glue( + all_message: &Ident, + wrapper: &Ident, + message_types: &[Path], +) -> Result { // The message types are path based, each of them must finish with a type // and as such we do this upfront. let variants: Vec<_> = Result::from_iter(message_types.into_iter().map(|path| { - let x = path + let ident = path .segments .last() .ok_or_else(|| { syn::Error::new(wrapper.span(), "Path is empty, but it must end with an identifier") }) .map(|segment| segment.ident.clone()); - x + ident }))?; let ts = quote! { #[allow(missing_docs)] @@ -83,7 +240,6 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu } )* }; - Ok(ts) } @@ -92,6 +248,7 @@ pub(crate) fn impl_subsystem_sender( wrapper_message: &Ident, support_crate: &TokenStream, outgoing_wrapper: &Ident, + all_messages_wrapper: &Ident, subsystem_sender_name: &Ident, ) -> TokenStream { let mut ts = quote! { @@ -106,39 +263,6 @@ pub(crate) fn impl_subsystem_sender( } }; - // Implement for all individual messages to avoid - // the necessity for manual wrapping, and to limit what a subsystem - // can actually send. This allows the generation of _the_ graph. - let sends = &ssf.messages_to_send; - ts.extend(quote!{ - #( - #[#support_crate ::async_trait] - impl SubsystemSender< #sends > for #subsystem_sender_name { - async fn send_message(&mut self, msg: #sends) { - self.channels.send_and_log_error( - self.signals_received.load(), - #wrapper_message ::from ( msg ) - ).await; - } - - async fn send_messages(&mut self, msgs: T) - where - T: IntoIterator + Send, - T::IntoIter: Send, - { - // TODO This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; - } - } - - fn send_unbounded_message(&mut self, msg: #sends) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )); - } - } - )* - }); - // Create the same for a wrapping enum: // // 1. subsystem specific `*OutgoingMessages`-type @@ -147,24 +271,38 @@ pub(crate) fn impl_subsystem_sender( quote! { /// implementation for wrapping message type... #[#support_crate ::async_trait] - impl SubsystemSender< #outgoing_wrapper > for #subsystem_sender_name { - async fn send_message(&mut self, msg: #outgoing_wrapper) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; + impl SubsystemSender< Message > for #subsystem_sender_name + where + Message: Send + 'static, + #outgoing_wrapper: ::std::convert::From, + #all_messages_wrapper: ::std::convert::From, + { + async fn send_message(&mut self, msg: Message) { + self.channels.send_and_log_error( + self.signals_received.load(), + #all_messages_wrapper ::from ( + #outgoing_wrapper :: from ( msg ) + ) + ).await; } - async fn send_messages(&mut self, msgs: T) + async fn send_messages(&mut self, msgs: I) where - T: IntoIterator + Send, - T::IntoIter: Send, + I: IntoIterator + Send, + I::IntoIter: Iterator, { - // This can definitely be optimized if necessary. for msg in msgs { - self.send_message(msg).await; + self.send_message( msg ); } } - fn send_unbounded_message(&mut self, msg: #outgoing_wrapper) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + fn send_unbounded_message(&mut self, msg: Message) { + self.channels.send_unbounded_and_log_error( + self.signals_received.load(), + #all_messages_wrapper ::from ( + #outgoing_wrapper :: from ( msg ) + ) + ); } } } @@ -174,7 +312,7 @@ pub(crate) fn impl_subsystem_sender( ts.extend(wrapped(&outgoing_wrapper)); // TODO FIXME - let inconsequent = true; + let inconsequent = false; if inconsequent { ts.extend(wrapped(wrapper_message)); } @@ -186,7 +324,8 @@ pub(crate) fn impl_subsystem_sender( /// which acts as the gateway to constructing the overseer. pub(crate) fn impl_subsystem_context( info: &OverseerInfo, - outgoing_messages: &Ident, + consumes: &Ident, + sends: &Ident, subsystem_sender_name: &Ident, subsystem_ctx_name: &Ident, ) -> TokenStream { @@ -204,23 +343,23 @@ pub(crate) fn impl_subsystem_context( /// [`SubsystemJob`]: trait.SubsystemJob.html #[derive(Debug)] #[allow(missing_docs)] - pub struct #subsystem_ctx_name{ + pub struct #subsystem_ctx_name { signals: #support_crate ::metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages, + messages: SubsystemIncomingMessages< #consumes >, to_subsystems: #subsystem_sender_name, to_overseer: #support_crate ::metered::UnboundedMeteredSender< #support_crate ::ToOverseer >, signals_received: SignalsReceived, - pending_incoming: Option<(usize, M)>, + pending_incoming: Option<(usize, #consumes)>, name: &'static str } - impl #subsystem_ctx_name { + impl #subsystem_ctx_name { /// Create a new context. fn new( signals: #support_crate ::metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages, + messages: SubsystemIncomingMessages< #consumes >, to_subsystems: ChannelsOut, to_overseer: #support_crate ::metered::UnboundedMeteredSender<#support_crate:: ToOverseer>, name: &'static str @@ -246,31 +385,30 @@ pub(crate) fn impl_subsystem_context( } #[#support_crate ::async_trait] - impl #support_crate ::SubsystemContext for #subsystem_ctx_name + impl #support_crate ::SubsystemContext for #subsystem_ctx_name where - #subsystem_sender_name: #support_crate ::SubsystemSender< #outgoing_messages >, - #outgoing_messages: From, + #subsystem_sender_name: #support_crate ::SubsystemSender< #sends >, { - type Message = M; + type Message = #consumes; type Signal = #signal; type Sender = #subsystem_sender_name; - type OutgoingMessages = #outgoing_messages; + type OutgoingMessages = #sends; type Error = #error_ty; - async fn try_recv(&mut self) -> ::std::result::Result>, ()> { + async fn try_recv(&mut self) -> ::std::result::Result>, ()> { match #support_crate ::poll!(self.recv()) { #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), #support_crate ::Poll::Pending => Ok(None), } } - async fn recv(&mut self) -> ::std::result::Result, #error_ty> { + async fn recv(&mut self) -> ::std::result::Result, #error_ty> { loop { // If we have a message pending an overseer signal, we only poll for signals // in the meantime. if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { if needs_signals_received <= self.signals_received.load() { - return Ok(#support_crate ::FromOverseer::Communication { msg }); + return Ok( #support_crate ::FromOverseer::Communication { msg }); } else { self.pending_incoming = Some((needs_signals_received, msg)); @@ -282,7 +420,7 @@ pub(crate) fn impl_subsystem_context( ))?; self.signals_received.inc(); - return Ok(#support_crate ::FromOverseer::Signal(signal)) + return Ok( #support_crate ::FromOverseer::Signal(signal)) } } @@ -295,7 +433,7 @@ pub(crate) fn impl_subsystem_context( let from_overseer = #support_crate ::futures::select_biased! { signal = await_signal => { let signal = signal - .ok_or(#support_crate ::OverseerError::Context( + .ok_or( #support_crate ::OverseerError::Context( "Signal channel is terminated and empty." .to_owned() ))?; @@ -304,7 +442,7 @@ pub(crate) fn impl_subsystem_context( } msg = await_message => { let packet = msg - .ok_or(#support_crate ::OverseerError::Context( + .ok_or( #support_crate ::OverseerError::Context( "Message channel is terminated and empty." .to_owned() ))?; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs index 7fd849916cb5..36c60cd84f60 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs @@ -178,12 +178,7 @@ impl Parse for Sends { let sends = Punctuated::parse_terminated(&content)?; (bracket, sends) }; - Ok(Self { - keyword_sends, - colon, - bracket, - sends, - }) + Ok(Self { keyword_sends, colon, bracket, sends }) } } diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index b133a28dd850..f9ef90577326 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -351,6 +351,14 @@ impl From for FromOverseer { } } +/// Binds a generated type which implements `#generated_outgoing: From` +/// for all annotated types. +/// +/// ``` +/// +/// ``` +pub trait AssociateOutgoing {} + /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or spawn jobs. @@ -374,7 +382,7 @@ pub trait SubsystemContext: Send + 'static { /// Try to asynchronously receive a message. /// - /// This has to be used with caution, if you loop over this without + /// Has to be used with caution, if you loop over this without /// using `pending!()` macro you will end up with a busy loop! async fn try_recv(&mut self) -> Result>, ()>; @@ -405,12 +413,12 @@ pub trait SubsystemContext: Send + 'static { } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) + async fn send_messages(&mut self, msgs: I) where - T: IntoIterator + Send, - T::IntoIter: Send, - Self::OutgoingMessages: From, - X: Send, + I: IntoIterator + Send, + I::IntoIter: Send, + Self::OutgoingMessages: From, + T: Send, { self.sender() .send_messages(msgs.into_iter().map(|x| ::from(x))) From 6306db88dc8156313bb2caff0401bb8b46a4ba7a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 20 Apr 2022 10:39:18 +0200 Subject: [PATCH 008/139] cleanup overseer a bit, make dummy work --- node/overseer/overseer-gen/examples/dummy.rs | 13 ++- .../overseer-gen/proc-macro/Cargo.toml | 3 + .../overseer/overseer-gen/proc-macro/build.rs | 3 + .../proc-macro/src/impl_subsystem.rs | 88 ++++++------------- 4 files changed, 44 insertions(+), 63 deletions(-) create mode 100644 node/overseer/overseer-gen/proc-macro/build.rs diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 1ec429161d00..8532539bc98d 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -9,8 +9,11 @@ use std::collections::HashMap; pub struct AwesomeSubSys; impl ::polkadot_overseer_gen::Subsystem for AwesomeSubSys { - fn start(self, ctx: AwesomeSubSysContext) -> SpawnedSubsystem { - ctx.spawn("awesome", Box::pin(async move { ctx.send_message(Plinko).await })); + fn start(self, mut ctx: AwesomeSubSysContext) -> SpawnedSubsystem { + let mut sender = ctx.sender().clone(); + + ctx.spawn("awesome", Box::pin(async move { sender.send_message(Plinko).await })) + .unwrap(); unimplemented!("starting yay!") } } @@ -19,8 +22,10 @@ impl ::polkadot_overseer_gen::Subsystem for Awesome pub struct GoblinTower; impl ::polkadot_overseer_gen::Subsystem for GoblinTower { - fn start(self, ctx: GoblinTowerContext) -> SpawnedSubsystem { - ctx.spawn("awesome", Box::pin(async move { ctx.send_message(MsgStrukt(0u8)).await })); + fn start(self, mut ctx: GoblinTowerContext) -> SpawnedSubsystem { + let mut sender = ctx.sender().clone(); + ctx.spawn("awesome", Box::pin(async move { sender.send_message(MsgStrukt(0u8)).await })) + .unwrap(); unimplemented!("welcum") } } diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml index 289542926338..38d82110bf1c 100644 --- a/node/overseer/overseer-gen/proc-macro/Cargo.toml +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -27,3 +27,6 @@ default = ["expand"] # write the expanded version to a `overlord-expansion.[a-f0-9]{10}.rs` # in the `OUT_DIR` as defined by `cargo` for the `expander` crate. expand = [] +# Create directional message consuming / outgoing graph. +# Generates: `${OUT_DIR}/${overseer|lowercase}-subsystem-messaging.dot` +graph = [] diff --git a/node/overseer/overseer-gen/proc-macro/build.rs b/node/overseer/overseer-gen/proc-macro/build.rs new file mode 100644 index 000000000000..24bd09048397 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/build.rs @@ -0,0 +1,3 @@ +fn main() { + // populate OUT_DIR +} diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 0556376b782d..fdf2bac42f35 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -43,7 +43,7 @@ fn graphviz( edge.weight().get_ident().expect("Must have a trailing identifier. qed") ) }, - &|_graph, (node_index, subsystem_name)| -> String { + &|_graph, (_node_index, subsystem_name)| -> String { format!(r#"label="{}""#, subsystem_name,) }, ); @@ -54,7 +54,8 @@ fn graphviz( pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let mut ts = TokenStream::new(); - let span = info.overseer_name.span(); + let overseer_name = &info.overseer_name; + let span = overseer_name.span(); let all_messages_wrapper = &info.message_wrapper; let support_crate = info.support_crate_name(); @@ -81,21 +82,22 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { consuming_lut.insert(&ssf.message_to_consume, (ssf.generic.clone(), node_index)); } - for (message_ty, (consuming_subsystem_ident, consuming_node_index)) in consuming_lut.iter() { + for (message_ty, (_consuming_subsystem_ident, consuming_node_index)) in consuming_lut.iter() { // match the outgoing ones that were registered above with the consumed message if let Some(origin_subsystems) = outgoing_lut.get(message_ty) { - for (origin_subsystem, sending_node_index) in origin_subsystems.iter() { + for (_origin_subsystem_ident, sending_node_index) in origin_subsystems.iter() { graph.add_edge(*sending_node_index, *consuming_node_index, (*message_ty).clone()); } } } - // all outgoing edges are now usable to deriver everything we need + // All outgoing edges are now usable to derive everything we need for node_index in graph.node_indices() { let subsystem_name = graph[node_index].to_string(); let outgoing_wrapper = Ident::new(&(subsystem_name + "OutgoingMessages"), span); // cannot be a hashmap, duplicate keys and sorting required + // maps outgoing messages to the subsystem that consumes it let outgoing_to_consumer = graph .edges_directed(node_index, Direction::Outgoing) .map(|edge| { @@ -105,6 +107,7 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { }) .collect::>>()?; + // Split it for usage with quote let outgoing_variant = outgoing_to_consumer.iter().map(|x| x.0.clone()).collect::>(); let subsystem_generic = outgoing_to_consumer.into_iter().map(|x| x.1).collect::>(); @@ -121,32 +124,37 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { }) } - if true { - let mut f = std::fs::OpenOptions::new() + // Dump the graph to file. + if cfg!(feature = "graph") { + let path = std::path::PathBuf::from( + std::env::var("OUT_DIR").expect("build.rs exists, hence OUT_DIR is populated. qed"), + ) + .join(overseer_name.to_string().to_lowercase() + "-subsystem-messaging.dot"); + if let Err(e) = std::fs::OpenOptions::new() .truncate(true) .create(true) .write(true) - .open("/tmp/foo.dot") - .expect("Opening that file works. qed"); - graphviz(&graph, &mut f).expect("Write of dot graph should work. qed"); + .open(&path) + .and_then(|mut f| graphviz(&graph, &mut f)) + { + eprintln!("Failed to write dot graph to {}: {:?}", path.display(), e); + } } + // Create all subsystem specific types, one by one for ssf in info.subsystems() { let subsystem_name = ssf.generic.to_string(); let subsystem_sender_name = Ident::new(&(subsystem_name.clone() + "Sender"), span); let subsystem_ctx_name = Ident::new(&(subsystem_name.clone() + "Context"), span); let outgoing_wrapper = Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span); - let consumes = ssf - .message_to_consume - .get_ident() - .cloned() - .unwrap_or_else(|| Ident::new("()", Span::call_site())); + let consumes = ssf.message_to_consume.get_ident().cloned().expect( + "Consumable message must have an trailing Ident which was checked in `Parse`. qed", + ); ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?); ts.extend(impl_subsystem_sender( - ssf, &all_messages_wrapper, support_crate, &outgoing_wrapper, @@ -164,6 +172,7 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { Ok(ts) } +/// Extract the final component of the message type path as used in the `#[subsystem(consumes: path::to::Foo)]` annotation. fn to_variant(path: &Path, span: Span) -> Result { let ident = path .segments @@ -205,46 +214,7 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu Ok(ts) } -/// Generates the wrapper type enum, no bells or whistles. -pub(crate) fn impl_wrapper_to_wrapper_glue( - all_message: &Ident, - wrapper: &Ident, - message_types: &[Path], -) -> Result { - // The message types are path based, each of them must finish with a type - // and as such we do this upfront. - let variants: Vec<_> = Result::from_iter(message_types.into_iter().map(|path| { - let ident = path - .segments - .last() - .ok_or_else(|| { - syn::Error::new(wrapper.span(), "Path is empty, but it must end with an identifier") - }) - .map(|segment| segment.ident.clone()); - ident - }))?; - let ts = quote! { - #[allow(missing_docs)] - #[derive(Clone)] - pub enum #wrapper { - #( - #variants ( #message_types ), - )* - } - - #( - impl ::std::convert::From< #message_types > for #wrapper { - fn from(message: #message_types) -> Self { - #wrapper :: #variants ( message ) - } - } - )* - }; - Ok(ts) -} - pub(crate) fn impl_subsystem_sender( - ssf: &SubSysField, wrapper_message: &Ident, support_crate: &TokenStream, outgoing_wrapper: &Ident, @@ -274,8 +244,8 @@ pub(crate) fn impl_subsystem_sender( impl SubsystemSender< Message > for #subsystem_sender_name where Message: Send + 'static, - #outgoing_wrapper: ::std::convert::From, - #all_messages_wrapper: ::std::convert::From, + #outgoing_wrapper: ::std::convert::From + Send, + #all_messages_wrapper: ::std::convert::From<#outgoing_wrapper> + Send, { async fn send_message(&mut self, msg: Message) { self.channels.send_and_log_error( @@ -289,10 +259,10 @@ pub(crate) fn impl_subsystem_sender( async fn send_messages(&mut self, msgs: I) where I: IntoIterator + Send, - I::IntoIter: Iterator, + I::IntoIter: Iterator + Send, { for msg in msgs { - self.send_message( msg ); + self.send_message( msg ).await; } } From 9053e17d53daa8f2fef87702964cc7a83727ff19 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 20 Apr 2022 18:40:26 +0200 Subject: [PATCH 009/139] more migration bits --- node/core/chain-selection/src/lib.rs | 2 +- node/overseer/examples/minimal-example.rs | 25 +-- node/overseer/overseer-gen/README.md | 4 +- node/overseer/overseer-gen/examples/dummy.rs | 37 ++-- .../overseer-gen/proc-macro/Cargo.toml | 2 +- .../proc-macro/src/impl_builder.rs | 19 +- .../proc-macro/src/impl_message_wrapper.rs | 3 +- .../proc-macro/src/impl_subsystem.rs | 174 +++++++++++------- node/overseer/overseer-gen/src/lib.rs | 31 ++-- node/overseer/src/dummy.rs | 10 +- node/overseer/src/lib.rs | 48 ++--- node/overseer/src/tests.rs | 32 +--- 12 files changed, 205 insertions(+), 182 deletions(-) diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index 0728ed23c043..b3136ecdae20 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -363,7 +363,7 @@ async fn run( match res { Err(e) => { e.trace(); - // All errors right now are considered fatal: + // All errors are considered fatal right now: break }, Ok(()) => { diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index fc07672ef4a1..524b425f461e 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -31,7 +31,7 @@ use polkadot_overseer::{ self as overseer, dummy::dummy_overseer_builder, gen::{FromOverseer, SpawnedSubsystem}, - AllMessages, HeadSupportsParachains, OverseerSignal, SubsystemError, + HeadSupportsParachains, OverseerSignal, SubsystemError, }; use polkadot_primitives::v2::{CandidateReceipt, Hash}; @@ -49,11 +49,7 @@ struct Subsystem1; impl Subsystem1 { async fn run(mut ctx: Ctx) -> () where - Ctx: overseer::SubsystemContext< - Message = CandidateBackingMessage, - AllMessages = AllMessages, - Signal = OverseerSignal, - >, + Ctx: overseer::SubsystemContext, { 'louy: loop { match ctx.try_recv().await { @@ -84,8 +80,7 @@ impl Subsystem1 { Default::default(), tx, ); - ctx.send_message(::AllMessages::from(msg)) - .await; + ctx.send_message(msg).await; } () } @@ -93,11 +88,7 @@ impl Subsystem1 { impl overseer::Subsystem for Subsystem1 where - Context: overseer::SubsystemContext< - Message = CandidateBackingMessage, - AllMessages = AllMessages, - Signal = OverseerSignal, - >, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { @@ -118,7 +109,6 @@ impl Subsystem2 { where Ctx: overseer::SubsystemContext< Message = CandidateValidationMessage, - AllMessages = AllMessages, Signal = OverseerSignal, >, { @@ -153,11 +143,8 @@ impl Subsystem2 { impl overseer::Subsystem for Subsystem2 where - Context: overseer::SubsystemContext< - Message = CandidateValidationMessage, - AllMessages = AllMessages, - Signal = OverseerSignal, - >, + Context: + overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { diff --git a/node/overseer/overseer-gen/README.md b/node/overseer/overseer-gen/README.md index cd1098a7a16c..ce03897f804c 100644 --- a/node/overseer/overseer-gen/README.md +++ b/node/overseer/overseer-gen/README.md @@ -63,10 +63,10 @@ is not ready to be included in the Overseer: ```rust #[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)] pub struct Overseer { - #[subsystem(MsgA)] + #[subsystem(MsgA, sends: MsgB)] sub_a: AwesomeSubSysA, - #[subsystem(MsgB), wip] + #[subsystem(MsgB, sends: MsgA), wip] sub_b: AwesomeSubSysB, // This subsystem will not be required nor allowed to be set } ``` diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 8532539bc98d..31b1c4caccf8 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -1,19 +1,23 @@ //! A dummy to be used with cargo expand use polkadot_node_network_protocol::WrongVariant; -use polkadot_overseer_gen::*; +use polkadot_overseer_gen::{SpawnNamed, *}; use std::collections::HashMap; /// Concrete subsystem implementation for `MsgStrukt` msg type. #[derive(Default)] pub struct AwesomeSubSys; -impl ::polkadot_overseer_gen::Subsystem for AwesomeSubSys { - fn start(self, mut ctx: AwesomeSubSysContext) -> SpawnedSubsystem { +impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { + fn start(self, mut ctx: XxxSubsystemContext) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); - - ctx.spawn("awesome", Box::pin(async move { sender.send_message(Plinko).await })) - .unwrap(); + ctx.spawn( + "AwesomeSubsys", + Box::pin(async move { + sender.send_message(Plinko).await; + }), + ) + .unwrap(); unimplemented!("starting yay!") } } @@ -21,11 +25,16 @@ impl ::polkadot_overseer_gen::Subsystem for Awesome #[derive(Default)] pub struct GoblinTower; -impl ::polkadot_overseer_gen::Subsystem for GoblinTower { - fn start(self, mut ctx: GoblinTowerContext) -> SpawnedSubsystem { +impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { + fn start(self, mut ctx: XxxSubsystemContext) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); - ctx.spawn("awesome", Box::pin(async move { sender.send_message(MsgStrukt(0u8)).await })) - .unwrap(); + ctx.spawn( + "GoblinTower", + Box::pin(async move { + sender.send_message(MsgStrukt(8u8)).await; + }), + ) + .unwrap(); unimplemented!("welcum") } } @@ -96,11 +105,11 @@ impl NetworkMsg { } #[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] -struct Yyy { - #[subsystem(consumes: MsgStrukt, sends: Plinko)] +struct Xxx { + #[subsystem(consumes: MsgStrukt, sends: [Plinko])] sub0: AwesomeSubSys, - #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: MsgStrukt)] + #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: [MsgStrukt])] plinkos: GoblinTower, i_like_pi: f64, @@ -135,7 +144,7 @@ impl SpawnNamed for DummySpawner { struct DummyCtx; fn main() { - let (overseer, _handle): (Yyy<_, f64>, _) = Yyy::builder() + let (overseer, _handle): (Xxx<_, f64>, _) = Xxx::builder() .sub0(AwesomeSubSys::default()) .plinkos(GoblinTower::default()) .i_like_pi(::std::f64::consts::PI) diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml index 38d82110bf1c..e1690807a7c9 100644 --- a/node/overseer/overseer-gen/proc-macro/Cargo.toml +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -23,7 +23,7 @@ petgraph = "0.6.0" assert_matches = "1.5.0" [features] -default = ["expand"] +default = ["graph", "expand"] # write the expanded version to a `overlord-expansion.[a-f0-9]{10}.rs` # in the `OUT_DIR` as defined by `cargo` for the `expander` crate. expand = [] diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 6c33a385efd0..93d45bfb9b76 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -35,11 +35,10 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { let builder = format_ident!("{}Builder", overseer_name); let handle = format_ident!("{}Handle", overseer_name); let connector = format_ident!("{}Connector", overseer_name); + let subsystem_ctx_name = format_ident!("{}SubsystemContext", overseer_name); let subsystem_name = &info.subsystem_names_without_wip(); let subsystem_generics = &info.subsystem_generic_types(); - let subsystem_ctx_names = - &Vec::from_iter(subsystem_generics.iter().map(|name| format_ident!("{}Context", name))); let consumes = &info.consumes_without_wip(); let channel_name = &info.channel_names_without_wip(""); @@ -104,8 +103,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { info.subsystems().iter().filter(|ssf| !ssf.wip).enumerate().map(|(idx, ssf)| { let field_name = &ssf.name; let field_type = &ssf.generic; - let subsystem_ctx_name = format_ident!("{}Context", &field_type); - + let subsystem_consumes = &ssf.message_to_consume; // Remove state generic for the item to be replaced. It sufficient to know `field_type` for // that since we always move from `Init<#field_type>` to `Init`. let impl_subsystem_state_generics = recollect_without_idx(&subsystem_passthrough_state_generics[..], idx); @@ -137,7 +135,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { impl #builder where - #field_type : Subsystem<#subsystem_ctx_name, #error_ty>, + #field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>, { /// Specify the subsystem in the builder directly pub fn #field_name (self, var: #field_type ) -> @@ -157,7 +155,6 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { signal_capacity: self.signal_capacity, } } - /// Specify the the initialization function for a subsystem pub fn #field_name_with<'a, F>(self, subsystem_init_fn: F ) -> #builder @@ -188,7 +185,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { impl #builder where - #field_type : Subsystem<#subsystem_ctx_name, #error_ty>, + #field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>, { /// Replace a subsystem by another implementation for the /// consumable message type. @@ -197,7 +194,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { where #field_type: 'static, F: 'static + FnOnce(#field_type) -> NEW, - NEW: #support_crate ::Subsystem<#subsystem_ctx_name, #error_ty>, + NEW: #support_crate ::Subsystem<#subsystem_ctx_name< #subsystem_consumes >, #error_ty>, { let replacement: Init = match self.#field_name { Init::Fn(fx) => @@ -341,7 +338,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { #builder #(, Missing<#field_type> )* > where #( - #subsystem_generics : Subsystem<#subsystem_ctx_names, #error_ty>, + #subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>, )* { #builder :: new() @@ -502,7 +499,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { where #spawner_where_clause, #( - #subsystem_generics : Subsystem<#subsystem_ctx_names, #error_ty>, + #subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>, )* { /// Complete the construction and create the overseer type. @@ -585,7 +582,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { // Convert owned `snake case` string to a `kebab case` static str. let subsystem_static_str = Box::leak(subsystem_string.replace("_", "-").into_boxed_str()); - let ctx = #subsystem_ctx_names::new( + let ctx = #subsystem_ctx_name::< #consumes >::new( signal_rx, message_rx, channels_out.clone(), diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 0be039b78671..7d29ddd458ab 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -52,7 +52,8 @@ pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result Result { #( #outgoing_wrapper :: #outgoing_variant ( msg ) => #all_messages_wrapper :: #subsystem_generic ( msg ), )* + #outgoing_wrapper :: Empty => #all_messages_wrapper :: Empty, } } } @@ -125,11 +126,9 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { } // Dump the graph to file. - if cfg!(feature = "graph") { - let path = std::path::PathBuf::from( - std::env::var("OUT_DIR").expect("build.rs exists, hence OUT_DIR is populated. qed"), - ) - .join(overseer_name.to_string().to_lowercase() + "-subsystem-messaging.dot"); + if cfg!(feature = "graph") || true { + let path = std::path::PathBuf::from(env!("OUT_DIR")) + .join(overseer_name.to_string().to_lowercase() + "-subsystem-messaging.dot"); if let Err(e) = std::fs::OpenOptions::new() .truncate(true) .create(true) @@ -138,36 +137,36 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { .and_then(|mut f| graphviz(&graph, &mut f)) { eprintln!("Failed to write dot graph to {}: {:?}", path.display(), e); + } else { + println!("Wrote dot graph to {}", path.display()); } } + let subsystem_sender_name = Ident::new(&(overseer_name.to_string() + "Sender"), span); + let subsystem_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), span); + ts.extend(impl_subsystem_context(info, &subsystem_sender_name, &subsystem_ctx_name)); + + ts.extend(impl_associate_outgoing_messages_trait(&all_messages_wrapper)); + + ts.extend(impl_subsystem_sender( + support_crate, + info.subsystems().iter().map(|ssf| { + let outgoing_wrapper = + Ident::new(&(ssf.generic.to_string() + "OutgoingMessages"), span); + outgoing_wrapper + }), + &all_messages_wrapper, + &subsystem_sender_name, + )); + // Create all subsystem specific types, one by one for ssf in info.subsystems() { let subsystem_name = ssf.generic.to_string(); - let subsystem_sender_name = Ident::new(&(subsystem_name.clone() + "Sender"), span); - let subsystem_ctx_name = Ident::new(&(subsystem_name.clone() + "Context"), span); - let outgoing_wrapper = Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span); - let consumes = ssf.message_to_consume.get_ident().cloned().expect( - "Consumable message must have an trailing Ident which was checked in `Parse`. qed", - ); - ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?); + ts.extend(impl_associate_outgoing_messages(&ssf.message_to_consume, &outgoing_wrapper)); - ts.extend(impl_subsystem_sender( - &all_messages_wrapper, - support_crate, - &outgoing_wrapper, - &all_messages_wrapper, - &subsystem_sender_name, - )); - ts.extend(impl_subsystem_context( - info, - &consumes, - &outgoing_wrapper, - &subsystem_sender_name, - &subsystem_ctx_name, - )); + ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?); } Ok(ts) } @@ -196,11 +195,12 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu let ts = quote! { #[allow(missing_docs)] - #[derive(Clone)] + #[derive(Debug)] pub enum #wrapper { #( #variants ( #message_types ), )* + Empty, } #( @@ -210,14 +210,20 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu } } )* + + // Useful for unit and integration tests: + impl ::std::convert::From< () > for #wrapper { + fn from(_message: ()) -> Self { + #wrapper :: Empty + } + } }; Ok(ts) } pub(crate) fn impl_subsystem_sender( - wrapper_message: &Ident, support_crate: &TokenStream, - outgoing_wrapper: &Ident, + outgoing_wrappers: impl IntoIterator, all_messages_wrapper: &Ident, subsystem_sender_name: &Ident, ) -> TokenStream { @@ -225,11 +231,13 @@ pub(crate) fn impl_subsystem_sender( /// Connector to send messages towards all subsystems, /// while tracking the which signals where already received. #[derive(Debug, Clone)] - pub struct #subsystem_sender_name { + pub struct #subsystem_sender_name < OutgoingWrapper > { /// Collection of channels to all subsystems. channels: ChannelsOut, /// Systemwide tick for which signals were received by all subsystems. signals_received: SignalsReceived, + /// Keep that marker around. + _phantom: ::core::marker::PhantomData< OutgoingWrapper >, } }; @@ -239,15 +247,15 @@ pub(crate) fn impl_subsystem_sender( // 2. overseer-global-`AllMessages`-type let wrapped = |outgoing_wrapper: &Ident| { quote! { - /// implementation for wrapping message type... + /// `M` references the type _consumed_ by the subsystem. #[#support_crate ::async_trait] - impl SubsystemSender< Message > for #subsystem_sender_name + impl SubsystemSender< OutgoingMessage > for #subsystem_sender_name < #outgoing_wrapper > where - Message: Send + 'static, - #outgoing_wrapper: ::std::convert::From + Send, - #all_messages_wrapper: ::std::convert::From<#outgoing_wrapper> + Send, + OutgoingMessage: ::std::fmt::Debug + Send + 'static, + #outgoing_wrapper: ::std::convert::From + Send, + #all_messages_wrapper: ::std::convert::From< #outgoing_wrapper > + Send, { - async fn send_message(&mut self, msg: Message) { + async fn send_message(&mut self, msg: OutgoingMessage) { self.channels.send_and_log_error( self.signals_received.load(), #all_messages_wrapper ::from ( @@ -258,15 +266,15 @@ pub(crate) fn impl_subsystem_sender( async fn send_messages(&mut self, msgs: I) where - I: IntoIterator + Send, - I::IntoIter: Iterator + Send, + I: IntoIterator + Send, + I::IntoIter: Iterator + Send, { for msg in msgs { self.send_message( msg ).await; } } - fn send_unbounded_message(&mut self, msg: Message) { + fn send_unbounded_message(&mut self, msg: OutgoingMessage) { self.channels.send_unbounded_and_log_error( self.signals_received.load(), #all_messages_wrapper ::from ( @@ -278,27 +286,55 @@ pub(crate) fn impl_subsystem_sender( } }; - // Allow the `#wrapper_message` to be sent as well. - ts.extend(wrapped(&outgoing_wrapper)); - - // TODO FIXME - let inconsequent = false; - if inconsequent { - ts.extend(wrapped(wrapper_message)); + for outgoing_wrapper in outgoing_wrappers { + ts.extend(wrapped(&outgoing_wrapper)); } ts } +pub(crate) fn impl_associate_outgoing_messages_trait(all_messages_wrapper: &Ident) -> TokenStream { + quote! { + /// Binds a generated type covering all declared outgoing messages, + /// which implements `#generated_outgoing: From` for all annotated types + /// of a particular subsystem. + /// + /// Note: This works because there is a 1?:1 relation between consumed messages and subsystems. + pub trait AssociateOutgoing: ::std::fmt::Debug + Send { + /// The associated _outgoing_ messages for a subsystem that _consumes_ the message `Self`. + type OutgoingMessages: Into< #all_messages_wrapper > + ::std::fmt::Debug + Send; + } + + impl AssociateOutgoing for () { + type OutgoingMessages = (); + } + } +} + +/// A workaround until default associated types actually work. +pub(crate) fn impl_associate_outgoing_messages( + message: &Path, + outgoing_wrapper: &Ident, +) -> TokenStream { + quote! { + impl AssociateOutgoing for #outgoing_wrapper { + type OutgoingMessages = #outgoing_wrapper; + } + + impl AssociateOutgoing for #message { + type OutgoingMessages = #outgoing_wrapper; + } + } +} + /// Implement a builder pattern for the `Overseer`-type, /// which acts as the gateway to constructing the overseer. pub(crate) fn impl_subsystem_context( info: &OverseerInfo, - consumes: &Ident, - sends: &Ident, subsystem_sender_name: &Ident, subsystem_ctx_name: &Ident, ) -> TokenStream { + let all_messages_wrapper = &info.message_wrapper; let signal = &info.extern_signal_ty; let error_ty = &info.extern_error_ty; let support_crate = info.support_crate_name(); @@ -313,34 +349,38 @@ pub(crate) fn impl_subsystem_context( /// [`SubsystemJob`]: trait.SubsystemJob.html #[derive(Debug)] #[allow(missing_docs)] - pub struct #subsystem_ctx_name { + pub struct #subsystem_ctx_name { signals: #support_crate ::metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages< #consumes >, - to_subsystems: #subsystem_sender_name, + messages: SubsystemIncomingMessages< M >, + to_subsystems: #subsystem_sender_name < ::OutgoingMessages >, to_overseer: #support_crate ::metered::UnboundedMeteredSender< #support_crate ::ToOverseer >, signals_received: SignalsReceived, - pending_incoming: Option<(usize, #consumes)>, + pending_incoming: Option<(usize, M)>, name: &'static str } - impl #subsystem_ctx_name { + impl #subsystem_ctx_name + where + M: AssociateOutgoing + Send + 'static, + { /// Create a new context. fn new( signals: #support_crate ::metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages< #consumes >, + messages: SubsystemIncomingMessages< M >, to_subsystems: ChannelsOut, to_overseer: #support_crate ::metered::UnboundedMeteredSender<#support_crate:: ToOverseer>, name: &'static str ) -> Self { let signals_received = SignalsReceived::default(); - #subsystem_ctx_name { + #subsystem_ctx_name :: { signals, messages, - to_subsystems: #subsystem_sender_name { + to_subsystems: #subsystem_sender_name :: < ::OutgoingMessages > { channels: to_subsystems, signals_received: signals_received.clone(), + _phantom: ::core::marker::PhantomData::default(), }, to_overseer, signals_received, @@ -355,24 +395,32 @@ pub(crate) fn impl_subsystem_context( } #[#support_crate ::async_trait] - impl #support_crate ::SubsystemContext for #subsystem_ctx_name + impl #support_crate ::SubsystemContext for #subsystem_ctx_name where - #subsystem_sender_name: #support_crate ::SubsystemSender< #sends >, + M: AssociateOutgoing, + #all_messages_wrapper: From, + #subsystem_sender_name< + ::OutgoingMessages + >: + #support_crate ::SubsystemSender< + ::OutgoingMessages + >, { - type Message = #consumes; + type Message = M; type Signal = #signal; - type Sender = #subsystem_sender_name; - type OutgoingMessages = #sends; + type OutgoingMessages = ::OutgoingMessages; + // type AllMessages = #all_messages_wrapper; + type Sender = #subsystem_sender_name < ::OutgoingMessages >; type Error = #error_ty; - async fn try_recv(&mut self) -> ::std::result::Result>, ()> { + async fn try_recv(&mut self) -> ::std::result::Result>, ()> { match #support_crate ::poll!(self.recv()) { #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), #support_crate ::Poll::Pending => Ok(None), } } - async fn recv(&mut self) -> ::std::result::Result, #error_ty> { + async fn recv(&mut self) -> ::std::result::Result, #error_ty> { loop { // If we have a message pending an overseer signal, we only poll for signals // in the meantime. diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index f9ef90577326..9af1b454a982 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -351,14 +351,6 @@ impl From for FromOverseer { } } -/// Binds a generated type which implements `#generated_outgoing: From` -/// for all annotated types. -/// -/// ``` -/// -/// ``` -pub trait AssociateOutgoing {} - /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or spawn jobs. @@ -374,7 +366,11 @@ pub trait SubsystemContext: Send + 'static { /// And the same for signals. type Signal: std::fmt::Debug + Send + 'static; /// The overarching messages `enum` for this particular subsystem. - type OutgoingMessages: Send + 'static; + type OutgoingMessages: std::fmt::Debug + Send + 'static; + + // The overarching messages `enum` for this particular subsystem. + // type AllMessages: From + From + std::fmt::Debug + Send + 'static; + /// The sender type as provided by `sender()` and underlying. type Sender: SubsystemSender + Send + 'static; /// The error type. @@ -404,10 +400,10 @@ pub trait SubsystemContext: Send + 'static { ) -> Result<(), Self::Error>; /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: X) + async fn send_message(&mut self, msg: T) where - Self::OutgoingMessages: From, - X: Send, + Self::OutgoingMessages: From, + T: Send, { self.sender().send_message(::from(msg)).await } @@ -457,14 +453,17 @@ where /// Sender end of a channel to interface with a subsystem. #[async_trait::async_trait] -pub trait SubsystemSender: Send + Clone + 'static { +pub trait SubsystemSender: Send + 'static +where + OutgoingMessage: Send, +{ /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: Message); + async fn send_message(&mut self, msg: OutgoingMessage); /// Send multiple direct messages to other `Subsystem`s, routed based on message type. async fn send_messages(&mut self, msgs: T) where - T: IntoIterator + Send, + T: IntoIterator + Send, T::IntoIter: Send; /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message @@ -472,7 +471,7 @@ pub trait SubsystemSender: Send + Clone + 'static { /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: Message); + fn send_unbounded_message(&mut self, msg: OutgoingMessage); } /// A future that wraps another future with a `Delay` allowing for time-limited futures. diff --git a/node/overseer/src/dummy.rs b/node/overseer/src/dummy.rs index 1b4dfc82a5d7..b4a97c3e6321 100644 --- a/node/overseer/src/dummy.rs +++ b/node/overseer/src/dummy.rs @@ -15,8 +15,8 @@ // along with Polkadot. If not, see . use crate::{ - prometheus::Registry, AllMessages, HeadSupportsParachains, InitializedOverseerBuilder, - MetricsTrait, Overseer, OverseerMetrics, OverseerSignal, OverseerSubsystemContext, SpawnNamed, + prometheus::Registry, HeadSupportsParachains, InitializedOverseerBuilder, MetricsTrait, + Overseer, OverseerMetrics, OverseerSignal, OverseerSubsystemContext, SpawnNamed, KNOWN_LEAVES_CACHE_SIZE, }; use lru::LruCache; @@ -30,11 +30,7 @@ pub struct DummySubsystem; impl Subsystem for DummySubsystem where - Context: SubsystemContext< - Signal = OverseerSignal, - Error = SubsystemError, - OutgoingMessages = DummySubsystemOutgoingMessages, - >, + Context: SubsystemContext, { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index fe7793643e45..cd1812f8be79 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -418,67 +418,73 @@ pub async fn forward_events>(client: Arc

, mut hand message_capacity=2048, )] pub struct Overseer { - #[subsystem(no_dispatch, CandidateValidationMessage)] + #[subsystem(no_dispatch, CandidateValidationMessage, sends: [])] candidate_validation: CandidateValidation, - #[subsystem(no_dispatch, PvfCheckerMessage)] + #[subsystem(no_dispatch, PvfCheckerMessage, sends: [])] pvf_checker: PvfChecker, - #[subsystem(no_dispatch, CandidateBackingMessage)] + #[subsystem(no_dispatch, CandidateBackingMessage, sends: [ + CandidateValidationMessage, + CollatorProtocolMessage, + AvailabilityDistributionMessage, + StatementDistributionMessage, + ProvisionerMessage, + ])] candidate_backing: CandidateBacking, - #[subsystem(StatementDistributionMessage)] + #[subsystem(StatementDistributionMessage, sends: [])] statement_distribution: StatementDistribution, - #[subsystem(no_dispatch, AvailabilityDistributionMessage)] + #[subsystem(no_dispatch, AvailabilityDistributionMessage, sends: [])] availability_distribution: AvailabilityDistribution, - #[subsystem(no_dispatch, AvailabilityRecoveryMessage)] + #[subsystem(no_dispatch, AvailabilityRecoveryMessage, sends: [])] availability_recovery: AvailabilityRecovery, - #[subsystem(blocking, no_dispatch, BitfieldSigningMessage)] + #[subsystem(blocking, no_dispatch, BitfieldSigningMessage, sends: [])] bitfield_signing: BitfieldSigning, - #[subsystem(BitfieldDistributionMessage)] + #[subsystem(BitfieldDistributionMessage, sends: [])] bitfield_distribution: BitfieldDistribution, - #[subsystem(no_dispatch, ProvisionerMessage)] + #[subsystem(no_dispatch, ProvisionerMessage, sends: [])] provisioner: Provisioner, - #[subsystem(no_dispatch, blocking, RuntimeApiMessage)] + #[subsystem(no_dispatch, blocking, RuntimeApiMessage, sends: [])] runtime_api: RuntimeApi, - #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage)] + #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage, sends: [])] availability_store: AvailabilityStore, - #[subsystem(no_dispatch, NetworkBridgeMessage)] + #[subsystem(no_dispatch, NetworkBridgeMessage, sends: [])] network_bridge: NetworkBridge, - #[subsystem(no_dispatch, blocking, ChainApiMessage)] + #[subsystem(no_dispatch, blocking, ChainApiMessage, sends: [])] chain_api: ChainApi, - #[subsystem(no_dispatch, CollationGenerationMessage)] + #[subsystem(no_dispatch, CollationGenerationMessage, sends: [])] collation_generation: CollationGeneration, - #[subsystem(no_dispatch, CollatorProtocolMessage)] + #[subsystem(no_dispatch, CollatorProtocolMessage, sends: [])] collator_protocol: CollatorProtocol, - #[subsystem(ApprovalDistributionMessage)] + #[subsystem(ApprovalDistributionMessage, sends: [])] approval_distribution: ApprovalDistribution, - #[subsystem(no_dispatch, blocking, ApprovalVotingMessage)] + #[subsystem(no_dispatch, blocking, ApprovalVotingMessage, sends: [])] approval_voting: ApprovalVoting, - #[subsystem(GossipSupportMessage)] + #[subsystem(GossipSupportMessage, sends: [])] gossip_support: GossipSupport, - #[subsystem(no_dispatch, blocking, DisputeCoordinatorMessage)] + #[subsystem(no_dispatch, blocking, DisputeCoordinatorMessage, sends: [])] dispute_coordinator: DisputeCoordinator, - #[subsystem(no_dispatch, DisputeDistributionMessage)] + #[subsystem(no_dispatch, DisputeDistributionMessage, sends: [])] dispute_distribution: DisputeDistribution, - #[subsystem(no_dispatch, blocking, ChainSelectionMessage)] + #[subsystem(no_dispatch, blocking, ChainSelectionMessage, sends: [])] chain_selection: ChainSelection, /// External listeners waiting for a hash to be in the active-leave set. diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index 2d486b75d883..e58bf37d8ba7 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -60,11 +60,7 @@ struct TestSubsystem1(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem1 where - C: overseer::SubsystemContext< - Message = CandidateValidationMessage, - Signal = OverseerSignal, - AllMessages = AllMessages, - >, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0; @@ -93,11 +89,7 @@ struct TestSubsystem2(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem2 where - C: overseer::SubsystemContext< - Message = CandidateBackingMessage, - Signal = OverseerSignal, - AllMessages = AllMessages, - >, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let sender = self.0.clone(); @@ -143,11 +135,7 @@ struct ReturnOnStart; impl overseer::Subsystem for ReturnOnStart where - C: overseer::SubsystemContext< - Message = CandidateBackingMessage, - Signal = OverseerSignal, - AllMessages = AllMessages, - >, + C: overseer::SubsystemContext, { fn start(self, mut _ctx: C) -> SpawnedSubsystem { SpawnedSubsystem { @@ -316,11 +304,7 @@ struct TestSubsystem5(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem5 where - C: overseer::SubsystemContext< - Message = CandidateValidationMessage, - Signal = OverseerSignal, - AllMessages = AllMessages, - >, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0.clone(); @@ -352,11 +336,7 @@ struct TestSubsystem6(metered::MeteredSender); impl Subsystem for TestSubsystem6 where - C: overseer::SubsystemContext< - Message = CandidateBackingMessage, - Signal = OverseerSignal, - AllMessages = AllMessages, - >, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0.clone(); @@ -761,7 +741,7 @@ impl CounterSubsystem { impl Subsystem for CounterSubsystem where - C: overseer::SubsystemContext, + C: overseer::SubsystemContext, M: Send, { fn start(self, mut ctx: C) -> SpawnedSubsystem { From d92a83ef61925f7fc7a8031b5e7024d9703a701f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 21 Apr 2022 11:55:46 +0200 Subject: [PATCH 010/139] fixups --- .../proc-macro/src/impl_subsystem.rs | 48 ++++++++++++++----- node/overseer/overseer-gen/src/lib.rs | 44 +++++++++-------- node/overseer/src/tests.rs | 6 ++- 3 files changed, 64 insertions(+), 34 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 8db7b1c2b579..cd86a1d64668 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -230,7 +230,7 @@ pub(crate) fn impl_subsystem_sender( let mut ts = quote! { /// Connector to send messages towards all subsystems, /// while tracking the which signals where already received. - #[derive(Debug, Clone)] + #[derive(Debug)] pub struct #subsystem_sender_name < OutgoingWrapper > { /// Collection of channels to all subsystems. channels: ChannelsOut, @@ -239,33 +239,49 @@ pub(crate) fn impl_subsystem_sender( /// Keep that marker around. _phantom: ::core::marker::PhantomData< OutgoingWrapper >, } + + // can't derive due to `PhantomData` and `OutgoingWrapper` not being + // bounded by `Clone`. + impl std::clone::Clone for #subsystem_sender_name < OutgoingWrapper > { + fn clone(&self) -> Self { + Self { + channels: self.channels.clone(), + signals_received: self.signals_received.clone(), + _phantom: ::core::marker::PhantomData::default(), + } + } + } }; // Create the same for a wrapping enum: // // 1. subsystem specific `*OutgoingMessages`-type // 2. overseer-global-`AllMessages`-type - let wrapped = |outgoing_wrapper: &Ident| { + let wrapped = |outgoing_wrapper: &TokenStream| { quote! { /// `M` references the type _consumed_ by the subsystem. #[#support_crate ::async_trait] - impl SubsystemSender< OutgoingMessage > for #subsystem_sender_name < #outgoing_wrapper > + impl SubsystemSender< #outgoing_wrapper > for #subsystem_sender_name < #outgoing_wrapper > where - OutgoingMessage: ::std::fmt::Debug + Send + 'static, - #outgoing_wrapper: ::std::convert::From + Send, #all_messages_wrapper: ::std::convert::From< #outgoing_wrapper > + Send, { - async fn send_message(&mut self, msg: OutgoingMessage) { + async fn send_message(&mut self, msg: OutgoingMessage) + where + OutgoingMessage: Send, + #outgoing_wrapper: ::std::convert::From + Send, + { self.channels.send_and_log_error( self.signals_received.load(), - #all_messages_wrapper ::from ( - #outgoing_wrapper :: from ( msg ) + <#all_messages_wrapper as ::std::convert::From<_>> ::from ( + <#outgoing_wrapper as ::std::convert::From<_>> :: from ( msg ) ) ).await; } - async fn send_messages(&mut self, msgs: I) + async fn send_messages(&mut self, msgs: I) where + OutgoingMessage: Send, + #outgoing_wrapper: ::std::convert::From + Send, I: IntoIterator + Send, I::IntoIter: Iterator + Send, { @@ -274,11 +290,15 @@ pub(crate) fn impl_subsystem_sender( } } - fn send_unbounded_message(&mut self, msg: OutgoingMessage) { + fn send_unbounded_message(&mut self, msg: OutgoingMessage) + where + OutgoingMessage: Send, + #outgoing_wrapper: ::std::convert::From + Send, + { self.channels.send_unbounded_and_log_error( self.signals_received.load(), - #all_messages_wrapper ::from ( - #outgoing_wrapper :: from ( msg ) + <#all_messages_wrapper as ::std::convert::From<_>> ::from ( + <#outgoing_wrapper as ::std::convert::From<_>> :: from ( msg ) ) ); } @@ -287,9 +307,11 @@ pub(crate) fn impl_subsystem_sender( }; for outgoing_wrapper in outgoing_wrappers { - ts.extend(wrapped(&outgoing_wrapper)); + ts.extend(wrapped("e! { #outgoing_wrapper })); } + ts.extend(wrapped("e! { () })); + ts } diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 9af1b454a982..1b477519103d 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -362,11 +362,11 @@ pub trait SubsystemContext: Send + 'static { /// The message type of this context. Subsystems launched with this context will expect /// to receive messages of this type. Commonly uses the wrapping `enum` commonly called /// `AllMessages`. - type Message: std::fmt::Debug + Send + 'static; + type Message: ::std::fmt::Debug + Send + 'static; /// And the same for signals. - type Signal: std::fmt::Debug + Send + 'static; + type Signal: ::std::fmt::Debug + Send + 'static; /// The overarching messages `enum` for this particular subsystem. - type OutgoingMessages: std::fmt::Debug + Send + 'static; + type OutgoingMessages: ::std::fmt::Debug + Send + 'static; // The overarching messages `enum` for this particular subsystem. // type AllMessages: From + From + std::fmt::Debug + Send + 'static; @@ -402,32 +402,31 @@ pub trait SubsystemContext: Send + 'static { /// Send a direct message to some other `Subsystem`, routed based on message type. async fn send_message(&mut self, msg: T) where - Self::OutgoingMessages: From, + Self::OutgoingMessages: From + Send, T: Send, { - self.sender().send_message(::from(msg)).await + self.sender().send_message::(msg).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. async fn send_messages(&mut self, msgs: I) where + Self::OutgoingMessages: From + Send, I: IntoIterator + Send, I::IntoIter: Send, - Self::OutgoingMessages: From, T: Send, { - self.sender() - .send_messages(msgs.into_iter().map(|x| ::from(x))) - .await + self.sender().send_messages::(msgs.into_iter()).await } /// Send a message using the unbounded connection. fn send_unbounded_message(&mut self, msg: X) where - Self::OutgoingMessages: From, + Self::OutgoingMessages: From + Send, X: Send, { - self.sender().send_unbounded_message(Self::OutgoingMessages::from(msg)) + self.sender() + .send_unbounded_message::(Self::OutgoingMessages::from(msg)) } /// Obtain the sender. @@ -453,25 +452,30 @@ where /// Sender end of a channel to interface with a subsystem. #[async_trait::async_trait] -pub trait SubsystemSender: Send + 'static -where - OutgoingMessage: Send, -{ +pub trait SubsystemSender: Send + 'static { /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: OutgoingMessage); + async fn send_message(&mut self, msg: OutgoingMessage) + where + OutgoingMessages: From, + OutgoingMessage: Send; /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) + async fn send_messages(&mut self, msgs: I) where - T: IntoIterator + Send, - T::IntoIter: Send; + OutgoingMessages: From, + OutgoingMessage: Send, + I: IntoIterator + Send, + I::IntoIter: Send; /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message /// type. /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: OutgoingMessage); + fn send_unbounded_message(&mut self, msg: OutgoingMessage) + where + OutgoingMessages: From, + OutgoingMessage: Send; } /// A future that wraps another future with a `Delay` allowing for time-limited futures. diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index e58bf37d8ba7..9fb030140191 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -89,7 +89,11 @@ struct TestSubsystem2(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem2 where - C: overseer::SubsystemContext, + C: overseer::SubsystemContext< + Message = CandidateBackingMessage, + OutgoingMessages = ::OutgoingMessages, + Signal = OverseerSignal, + >, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let sender = self.0.clone(); From f3ad18dfe9c1fe2104c87b7a0f889b31fb63973f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 21 Apr 2022 12:32:52 +0200 Subject: [PATCH 011/139] fixup sample --- node/overseer/examples/minimal-example.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 524b425f461e..2bd9cbadbc46 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -49,7 +49,11 @@ struct Subsystem1; impl Subsystem1 { async fn run(mut ctx: Ctx) -> () where - Ctx: overseer::SubsystemContext, + Ctx: overseer::SubsystemContext< + Message = CandidateBackingMessage, + Signal = OverseerSignal + >, + ::OutgoingMessages: From, { 'louy: loop { match ctx.try_recv().await { @@ -88,7 +92,11 @@ impl Subsystem1 { impl overseer::Subsystem for Subsystem1 where - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CandidateBackingMessage, + OutgoingMessages = overseer::CandidateBackingOutgoingMessages, + Signal = OverseerSignal + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { From 6cb6540001a7324392c796cd71773463fc292efb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 21 Apr 2022 18:09:47 +0200 Subject: [PATCH 012/139] grunt work work --- node/collation-generation/src/lib.rs | 26 +++-- node/core/approval-voting/src/import.rs | 6 +- node/core/approval-voting/src/lib.rs | 44 +++++-- node/core/av-store/src/lib.rs | 48 ++++++-- node/core/bitfield-signing/src/lib.rs | 16 ++- node/core/candidate-validation/src/lib.rs | 28 +++-- node/core/chain-api/src/lib.rs | 16 ++- node/core/chain-selection/src/lib.rs | 24 +++- .../dispute-coordinator/src/initialized.rs | 40 +++++-- node/core/provisioner/src/lib.rs | 3 +- node/core/pvf-checker/src/lib.rs | 30 +++-- node/core/runtime-api/src/lib.rs | 16 ++- node/network/approval-distribution/src/lib.rs | 102 ++++++++++++----- .../availability-distribution/src/lib.rs | 16 ++- .../src/requester/mod.rs | 20 ++-- .../src/responder.rs | 16 +-- node/network/availability-recovery/src/lib.rs | 40 +++++-- node/network/bitfield-distribution/src/lib.rs | 29 ++--- node/network/bridge/src/lib.rs | 24 +++- .../src/collator_side/mod.rs | 100 +++++++++++----- node/network/collator-protocol/src/lib.rs | 19 ++- .../src/validator_side/mod.rs | 108 +++++++++++++----- node/network/dispute-distribution/src/lib.rs | 25 ++-- .../dispute-distribution/src/receiver/mod.rs | 3 +- node/network/gossip-support/src/lib.rs | 88 +++++++------- .../network/statement-distribution/src/lib.rs | 52 +++++---- node/overseer/examples/minimal-example.rs | 7 +- .../proc-macro/src/impl_subsystem.rs | 27 ++--- node/overseer/overseer-gen/src/lib.rs | 30 +++-- node/overseer/src/lib.rs | 18 ++- node/subsystem-test-helpers/src/lib.rs | 16 +-- .../src/determine_new_blocks.rs | 7 +- node/subsystem-util/src/lib.rs | 87 +++++++------- .../src/rolling_session_window.rs | 47 ++++---- node/subsystem-util/src/runtime/mod.rs | 41 +++---- node/subsystem-util/src/tests.rs | 1 + node/subsystem/src/lib.rs | 36 +----- 37 files changed, 783 insertions(+), 473 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index aee408637a99..a11440fd97f0 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -73,8 +73,12 @@ impl CollationGenerationSubsystem { /// Otherwise, most are logged and then discarded. async fn run(mut self, mut ctx: Context) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollationGenerationMessage, + OutgoingMessages = overseer::CollationGenerationOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { // when we activate new leaves, we spawn a bunch of sub-tasks, each of which is // expected to generate precisely one message. We don't want to block the main loop @@ -108,11 +112,13 @@ impl CollationGenerationSubsystem { &mut self, incoming: SubsystemResult::Message>>, ctx: &mut Context, - sender: &mpsc::Sender, + sender: &mpsc::Sender, ) -> bool where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollationGenerationMessage, + OutgoingMessages = overseer::CollationGenerationOutgoingMessages, + >, { match incoming { Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { @@ -164,8 +170,12 @@ impl CollationGenerationSubsystem { impl overseer::Subsystem for CollationGenerationSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollationGenerationMessage, + OutgoingMessages = overseer::CollationGenerationOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { @@ -417,7 +427,7 @@ async fn obtain_current_validation_code_hash( relay_parent: Hash, para_id: ParaId, assumption: OccupiedCoreAssumption, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result, crate::error::Error> { use polkadot_node_subsystem::RuntimeApiError; diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index 05df3997bfc2..d046d626480c 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -108,7 +108,7 @@ enum ImportedBlockInfoError { /// Computes information about the imported block. Returns an error if the info couldn't be extracted. async fn imported_block_info( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -320,7 +320,9 @@ pub struct BlockImportedCandidates { /// /// It is the responsibility of the caller to schedule wakeups for each block. pub(crate) async fn handle_new_head( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, + >, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, head: Hash, diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 7c3b1a0812cd..9ade68e48696 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -357,8 +357,12 @@ impl ApprovalVotingSubsystem { impl overseer::Subsystem for ApprovalVotingSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ApprovalVotingMessage, + OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); @@ -605,7 +609,7 @@ impl State { /// Bring `session_window` up to date. pub async fn cache_session_info_for_head( &mut self, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, head: Hash, ) -> Result, SessionsUnavailable> { let session_window = self.session_window.take(); @@ -709,8 +713,12 @@ async fn run( mut backend: B, ) -> SubsystemResult<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ApprovalVotingMessage, + OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, B: Backend, { let mut state = State { @@ -849,8 +857,12 @@ where // // returns `true` if any of the actions was a `Conclude` command. async fn handle_actions( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalVotingMessage, + OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, @@ -1087,8 +1099,12 @@ fn distribution_messages_for_activation( // Handle an incoming signal from the overseer. Returns true if execution should conclude. async fn handle_from_overseer( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalVotingMessage, + OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, @@ -1198,7 +1214,7 @@ async fn handle_from_overseer( } async fn handle_approved_ancestor( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, db: &OverlayedBackend<'_, impl Backend>, target: Hash, lower_bound: BlockNumber, @@ -2148,8 +2164,12 @@ fn process_wakeup( // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. async fn launch_approval( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalVotingMessage, + OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, metrics: Metrics, session_index: SessionIndex, candidate: CandidateReceipt, diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index dbb30fdda02a..20e055672dcd 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -521,8 +521,12 @@ impl KnownUnfinalizedBlocks { impl overseer::Subsystem for AvailabilityStoreSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityStoreMessage, + OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(self, ctx).map(|_| Ok(())).boxed(); @@ -533,8 +537,12 @@ where async fn run(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityStoreMessage, + OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let mut next_pruning = Delay::new(subsystem.pruning_config.pruning_interval).fuse(); @@ -562,8 +570,12 @@ async fn run_iteration( mut next_pruning: &mut future::Fuse, ) -> Result where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityStoreMessage, + OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { select! { incoming = ctx.recv().fuse() => { @@ -614,8 +626,12 @@ async fn process_block_activated( activated: Hash, ) -> Result<(), Error> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityStoreMessage, + OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let now = subsystem.clock.now()?; @@ -674,8 +690,12 @@ async fn process_new_head( header: Header, ) -> Result<(), Error> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityStoreMessage, + OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let candidate_events = util::request_candidate_events(hash, ctx.sender()).await.await??; @@ -821,8 +841,12 @@ async fn process_block_finalized( finalized_number: BlockNumber, ) -> Result<(), Error> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityStoreMessage, + OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let now = subsystem.clock.now()?; diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index 94b525ec6274..27284b44e40b 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -34,7 +34,7 @@ use polkadot_node_subsystem::{ AvailabilityStoreMessage, BitfieldDistributionMessage, BitfieldSigningMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, LeafStatus, PerLeafSpan, SubsystemSender, + overseer, ActivatedLeaf, LeafStatus, PerLeafSpan, SubsystemSender, }; use polkadot_node_subsystem_util::{self as util, JobSender, JobSubsystem, JobTrait, Validator}; use polkadot_primitives::v2::{AvailabilityBitfield, CoreState, Hash, ValidatorIndex}; @@ -83,7 +83,7 @@ pub enum Error { async fn get_core_availability( core: &CoreState, validator_idx: ValidatorIndex, - sender: &Mutex<&mut impl SubsystemSender>, + sender: &Mutex<&mut impl SubsystemSender>, span: &jaeger::Span, ) -> Result { if let &CoreState::Occupied(ref core) = core { @@ -122,7 +122,7 @@ async fn get_core_availability( /// delegates to the v1 runtime API async fn get_availability_cores( relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); sender @@ -146,7 +146,7 @@ async fn construct_availability_bitfield( relay_parent: Hash, span: &jaeger::Span, validator_idx: ValidatorIndex, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result { // get the set of availability cores from the runtime let availability_cores = { @@ -184,6 +184,7 @@ async fn construct_availability_bitfield( impl JobTrait for BitfieldSigningJob { type ToJob = BitfieldSigningMessage; + type OutgoingMessages = overseer::BitfieldSigningOutgoingMessages; type Error = Error; type RunArgs = SyncCryptoStorePtr; type Metrics = Metrics; @@ -191,13 +192,16 @@ impl JobTrait for BitfieldSigningJob { const NAME: &'static str = "bitfield-signing-job"; /// Run a job for the parent block indicated - fn run( + fn run( leaf: ActivatedLeaf, keystore: Self::RunArgs, metrics: Self::Metrics, _receiver: mpsc::Receiver, mut sender: JobSender, - ) -> Pin> + Send>> { + ) -> Pin> + Send>> + where + S: SubsystemSender, + { let metrics = metrics.clone(); async move { if let LeafStatus::Stale = leaf.status { diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 6da1bc0ac96c..aebf3c42b2bd 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -95,8 +95,12 @@ impl CandidateValidationSubsystem { impl overseer::Subsystem for CandidateValidationSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CandidateValidationMessage, + OutgoingMessages = overseer::CandidateValidationOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run( @@ -120,8 +124,12 @@ async fn run( program_path: PathBuf, ) -> SubsystemResult<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CandidateValidationMessage, + OutgoingMessages = overseer::CandidateValidationOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let (validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), @@ -235,7 +243,7 @@ async fn runtime_api_request( receiver: oneshot::Receiver>, ) -> Result where - Sender: SubsystemSender, + Sender: SubsystemSender, { sender .send_message(RuntimeApiMessage::Request(relay_parent, request).into()) @@ -268,7 +276,7 @@ async fn request_validation_code_by_hash( validation_code_hash: ValidationCodeHash, ) -> Result, RuntimeRequestFailed> where - Sender: SubsystemSender, + Sender: SubsystemSender, { let (tx, rx) = oneshot::channel(); runtime_api_request( @@ -287,7 +295,7 @@ async fn precheck_pvf( validation_code_hash: ValidationCodeHash, ) -> PreCheckOutcome where - Sender: SubsystemSender, + Sender: SubsystemSender, { let validation_code = match request_validation_code_by_hash(sender, relay_parent, validation_code_hash).await { @@ -342,7 +350,7 @@ async fn check_assumption_validation_data( assumption: OccupiedCoreAssumption, ) -> AssumptionCheckOutcome where - Sender: SubsystemSender, + Sender: SubsystemSender, { let validation_data = { let (tx, rx) = oneshot::channel(); @@ -386,7 +394,7 @@ async fn find_assumed_validation_data( descriptor: &CandidateDescriptor, ) -> AssumptionCheckOutcome where - Sender: SubsystemSender, + Sender: SubsystemSender, { // The candidate descriptor has a `persisted_validation_data_hash` which corresponds to // one of up to two possible values that we can derive from the state of the @@ -446,7 +454,7 @@ async fn validate_from_chain_state( metrics: &Metrics, ) -> Result where - Sender: SubsystemSender, + Sender: SubsystemSender, { let mut new_sender = sender.clone(); let (validation_data, validation_code) = diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 36282a2edccc..76d9ec697a66 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -67,8 +67,12 @@ impl ChainApiSubsystem { impl overseer::Subsystem for ChainApiSubsystem where Client: HeaderBackend + AuxStore + 'static, - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ChainApiMessage, + OutgoingMessages = overseer::ChainApiOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(ctx, self) @@ -84,8 +88,12 @@ async fn run( ) -> SubsystemResult<()> where Client: HeaderBackend + AuxStore, - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ChainApiMessage, + OutgoingMessages = overseer::ChainApiOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { loop { match ctx.recv().await? { diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index b3136ecdae20..febf349515d8 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -330,8 +330,12 @@ impl ChainSelectionSubsystem { impl overseer::Subsystem for ChainSelectionSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ChainSelectionMessage, + OutgoingMessages = overseer::ChainSelectionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = db_backend::v1::DbBackend::new( @@ -354,8 +358,12 @@ async fn run( stagnant_check_interval: StagnantCheckInterval, clock: Box, ) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ChainSelectionMessage, + OutgoingMessages = overseer::ChainSelectionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, B: Backend, { loop { @@ -386,8 +394,12 @@ async fn run_until_error( clock: &(dyn Clock + Sync), ) -> Result<(), Error> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ChainSelectionMessage, + OutgoingMessages = overseer::ChainSelectionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, B: Backend, { let mut stagnant_check_stream = stagnant_check_interval.timeout_stream(); diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index 40f9ed8211ff..d67ff56d7bf3 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -123,8 +123,12 @@ impl Initialized { clock: Box, ) -> FatalResult<()> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = DisputeCoordinatorMessage, + OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, B: Backend, { loop { @@ -161,8 +165,12 @@ impl Initialized { clock: &dyn Clock, ) -> Result<()> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = DisputeCoordinatorMessage, + OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, B: Backend, { for (priority, request) in participations.drain(..) { @@ -255,8 +263,12 @@ impl Initialized { async fn process_active_leaves_update( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = DisputeCoordinatorMessage, + OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, overlay_db: &mut OverlayedBackend<'_, impl Backend>, update: ActiveLeavesUpdate, now: u64, @@ -320,8 +332,12 @@ impl Initialized { /// relay chain. async fn process_on_chain_votes( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = DisputeCoordinatorMessage, + OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, overlay_db: &mut OverlayedBackend<'_, impl Backend>, votes: ScrapedOnChainVotes, now: u64, @@ -1056,8 +1072,12 @@ enum MuxedMessage { impl MuxedMessage { async fn receive( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = DisputeCoordinatorMessage, + OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, from_sender: &mut participation::WorkerMessageReceiver, ) -> FatalResult { // We are only fusing here to make `select` happy, in reality we will quit if the stream diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 4a33655274d8..6c2a69ebf29d 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -32,7 +32,7 @@ use polkadot_node_subsystem::{ CandidateBackingMessage, ChainApiMessage, DisputeCoordinatorMessage, ProvisionableData, ProvisionerInherentData, ProvisionerMessage, }, - ActivatedLeaf, LeafStatus, PerLeafSpan, SubsystemSender, + overseer, ActivatedLeaf, LeafStatus, PerLeafSpan, SubsystemSender, }; use polkadot_node_subsystem_util::{ request_availability_cores, request_persisted_validation_data, JobSender, JobSubsystem, @@ -112,6 +112,7 @@ pub struct ProvisionerConfig; impl JobTrait for ProvisionerJob { type ToJob = ProvisionerMessage; + type OutgoingMessages = overseer::ProvisionerOutgoingMessages; type Error = Error; type RunArgs = ProvisionerConfig; type Metrics = Metrics; diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index 5d6f544e39ae..e0cfde5e7b56 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -62,8 +62,12 @@ impl PvfCheckerSubsystem { impl overseer::Subsystem for PvfCheckerSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = PvfCheckerMessage, + OutgoingMessages = overseer::PvfCheckerOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { if self.enabled { @@ -129,8 +133,12 @@ async fn run( metrics: Metrics, ) -> SubsystemResult<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = PvfCheckerMessage, + OutgoingMessages = overseer::PvfCheckerOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let mut state = State { credentials: None, @@ -179,7 +187,7 @@ where /// Handle an incoming PVF pre-check result from the candidate-validation subsystem. async fn handle_pvf_check( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, metrics: &Metrics, outcome: PreCheckOutcome, @@ -247,7 +255,7 @@ struct Conclude; async fn handle_from_overseer( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, metrics: &Metrics, from_overseer: FromOverseer, @@ -273,7 +281,7 @@ async fn handle_from_overseer( async fn handle_leaves_update( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, metrics: &Metrics, update: ActiveLeavesUpdate, @@ -355,7 +363,7 @@ struct ActivationEffect { /// Returns `None` if the PVF pre-checking runtime API is not supported for the given leaf hash. async fn examine_activation( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, leaf_hash: Hash, leaf_number: BlockNumber, @@ -414,7 +422,7 @@ async fn examine_activation( /// Checks the active validators for the given leaf. If we have a signing key for one of them, /// returns the [`SigningCredentials`]. async fn check_signing_credentials( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, leaf: Hash, ) -> Option { @@ -443,7 +451,7 @@ async fn check_signing_credentials( /// /// If the validator already voted for the given code, this function does nothing. async fn sign_and_submit_pvf_check_statement( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, voted: &mut HashSet, credentials: &SigningCredentials, @@ -535,7 +543,7 @@ async fn sign_and_submit_pvf_check_statement( /// into the `currently_checking` set. async fn initiate_precheck( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, relay_parent: Hash, validation_code_hash: ValidationCodeHash, metrics: &Metrics, diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index fa20a2ad768a..4c7b0372d35c 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -96,8 +96,12 @@ impl overseer::Subsystem for RuntimeAp where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = RuntimeApiMessage, + OutgoingMessages = overseer::RuntimeApiOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { future: run(ctx, self).boxed(), name: "runtime-api-subsystem" } @@ -340,8 +344,12 @@ async fn run( where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = RuntimeApiMessage, + OutgoingMessages = overseer::RuntimeApiOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { loop { select! { diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index c2bbd1b4583b..eadab6f9c59a 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -324,8 +324,12 @@ enum PendingMessage { impl State { async fn handle_network_msg( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), @@ -379,8 +383,12 @@ impl State { async fn handle_new_blocks( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), @@ -522,8 +530,12 @@ impl State { async fn process_incoming_peer_message( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, metrics: &Metrics, peer_id: PeerId, msg: protocol_v1::ApprovalDistributionMessage, @@ -614,8 +626,12 @@ impl State { // and has an entry in the `PeerData` struct. async fn handle_peer_view_change( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, metrics: &Metrics, peer_id: PeerId, view: View, @@ -689,8 +705,12 @@ impl State { async fn import_and_circulate_assignment( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCert, @@ -948,8 +968,12 @@ impl State { async fn import_and_circulate_approval( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, @@ -1209,8 +1233,13 @@ impl State { } async fn unify_with_peer( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, + gossip_peers: &HashSet, metrics: &Metrics, entries: &mut HashMap, topologies: &SessionGridTopologies, @@ -1566,8 +1595,12 @@ async fn adjust_required_routing_and_propagate( /// Modify the reputation of a peer based on its behavior. async fn modify_reputation( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, peer_id: PeerId, rep: Rep, ) { @@ -1589,8 +1622,12 @@ impl ApprovalDistribution { async fn run(self, ctx: Context) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let mut state = State::default(); @@ -1605,10 +1642,15 @@ impl ApprovalDistribution { self, mut ctx: Context, state: &mut State, - rng: &mut (impl CryptoRng + Rng), - ) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + rng: &mut (impl CryptoRng + Rng) + ) + where + Context: overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { loop { let message = match ctx.recv().await { @@ -1645,8 +1687,12 @@ impl ApprovalDistribution { metrics: &Metrics, rng: &mut (impl CryptoRng + Rng), ) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { match msg { ApprovalDistributionMessage::NetworkBridgeUpdate(event) => { @@ -1692,8 +1738,12 @@ impl ApprovalDistribution { impl overseer::Subsystem for ApprovalDistribution where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = ApprovalDistributionMessage, + OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index f26a2c78e5d7..842a6e2e2f60 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -70,8 +70,12 @@ pub struct IncomingRequestReceivers { impl overseer::Subsystem for AvailabilityDistributionSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityDistributionMessage, + OutgoingMessages = overseer::AvailabilityDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -97,8 +101,12 @@ impl AvailabilityDistributionSubsystem { /// Start processing work as passed on from the Overseer. async fn run(self, mut ctx: Context) -> std::result::Result<(), FatalError> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityDistributionMessage, + OutgoingMessages = overseer::AvailabilityDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let Self { mut runtime, recvs, metrics } = self; diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index b1ea7a98c1f3..7a4043c01a9a 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -129,9 +129,10 @@ impl Requester { where Context: SubsystemContext, { + let mut sender = ctx.sender(); let ActivatedLeaf { hash: leaf, .. } = new_head; let (leaf_session_index, ancestors_in_session) = get_block_ancestors_in_same_session( - ctx, + &mut sender, runtime, leaf, Self::LEAF_ANCESTRY_LEN_WITHIN_SESSION, @@ -139,7 +140,7 @@ impl Requester { .await?; // Also spawn or bump tasks for candidates in ancestry in the same session. for hash in std::iter::once(leaf).chain(ancestors_in_session) { - let cores = get_occupied_cores(ctx, hash).await?; + let cores = get_occupied_cores(&mut sender, hash).await?; gum::trace!( target: LOG_TARGET, occupied_cores = ?cores, @@ -257,26 +258,27 @@ impl Stream for Requester { /// Requests up to `limit` ancestor hashes of relay parent in the same session. /// /// Also returns session index of the `head`. -async fn get_block_ancestors_in_same_session( - ctx: &mut Context, +async fn get_block_ancestors_in_same_session( + sender: &mut Sender, runtime: &mut RuntimeInfo, head: Hash, limit: usize, ) -> Result<(SessionIndex, Vec)> where - Context: SubsystemContext, + Sender: + overseer::SubsystemSender + overseer::SubsystemSender, { // The order is parent, grandparent, ... // // `limit + 1` since a session index for the last element in ancestry // is obtained through its parent. It always gets truncated because // `session_ancestry_len` can only be incremented `ancestors.len() - 1` times. - let mut ancestors = get_block_ancestors(ctx, head, limit + 1).await?; + let mut ancestors = get_block_ancestors(sender, head, limit + 1).await?; let mut ancestors_iter = ancestors.iter(); // `head` is the child of the first block in `ancestors`, request its session index. let head_session_index = match ancestors_iter.next() { - Some(parent) => runtime.get_session_index_for_child(ctx.sender(), *parent).await?, + Some(parent) => runtime.get_session_index_for_child(sender, *parent).await?, None => { // No first element, i.e. empty. return Ok((0, ancestors)) @@ -287,7 +289,7 @@ where // The first parent is skipped. for parent in ancestors_iter { // Parent is the i-th ancestor, request session index for its child -- (i-1)th element. - let session_index = runtime.get_session_index_for_child(ctx.sender(), *parent).await?; + let session_index = runtime.get_session_index_for_child(sender, *parent).await?; if session_index == head_session_index { session_ancestry_len += 1; } else { @@ -303,7 +305,7 @@ where /// Request up to `limit` ancestor hashes of relay parent from the Chain API. async fn get_block_ancestors( - ctx: &mut Context, + ctx: &mut overseer::SubsystemSender, relay_parent: Hash, limit: usize, ) -> Result> diff --git a/node/network/availability-distribution/src/responder.rs b/node/network/availability-distribution/src/responder.rs index 0520a86ca1d2..53c363823f72 100644 --- a/node/network/availability-distribution/src/responder.rs +++ b/node/network/availability-distribution/src/responder.rs @@ -43,7 +43,7 @@ pub async fn run_pov_receiver( mut receiver: IncomingRequestReceiver, metrics: Metrics, ) where - Sender: SubsystemSender, + Sender: SubsystemSender, { loop { match receiver.recv(|| vec![COST_INVALID_REQUEST]).await.into_nested() { @@ -71,7 +71,7 @@ pub async fn run_chunk_receiver( mut receiver: IncomingRequestReceiver, metrics: Metrics, ) where - Sender: SubsystemSender, + Sender: SubsystemSender, { loop { match receiver.recv(|| vec![COST_INVALID_REQUEST]).await.into_nested() { @@ -105,7 +105,7 @@ pub async fn answer_pov_request_log( req: IncomingRequest, metrics: &Metrics, ) where - Sender: SubsystemSender, + Sender: SubsystemSender, { let res = answer_pov_request(sender, req).await; match res { @@ -130,7 +130,7 @@ pub async fn answer_chunk_request_log( metrics: &Metrics, ) -> () where - Sender: SubsystemSender, + Sender: SubsystemSender, { let res = answer_chunk_request(sender, req).await; match res { @@ -154,7 +154,7 @@ pub async fn answer_pov_request( req: IncomingRequest, ) -> Result where - Sender: SubsystemSender, + Sender: SubsystemSender, { let _span = jaeger::Span::new(req.payload.candidate_hash, "answer-pov-request"); @@ -182,7 +182,7 @@ pub async fn answer_chunk_request( req: IncomingRequest, ) -> Result where - Sender: SubsystemSender, + Sender: SubsystemSender, { let span = jaeger::Span::new(req.payload.candidate_hash, "answer-chunk-request"); @@ -217,7 +217,7 @@ async fn query_chunk( validator_index: ValidatorIndex, ) -> std::result::Result, JfyiError> where - Sender: SubsystemSender, + Sender: SubsystemSender, { let (tx, rx) = oneshot::channel(); sender @@ -245,7 +245,7 @@ async fn query_available_data( candidate_hash: CandidateHash, ) -> Result> where - Sender: SubsystemSender, + Sender: SubsystemSender, { let (tx, rx) = oneshot::channel(); sender diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 533d4ec94f8d..d8bfd76cca60 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -801,8 +801,12 @@ impl Default for State { impl Subsystem for AvailabilityRecoverySubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityRecoveryMessage, + OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -842,8 +846,12 @@ async fn launch_recovery_task( metrics: &Metrics, ) -> error::Result<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityRecoveryMessage, + OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let candidate_hash = receipt.hash(); @@ -895,8 +903,12 @@ async fn handle_recover( metrics: &Metrics, ) -> error::Result<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityRecoveryMessage, + OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let candidate_hash = receipt.hash(); @@ -958,8 +970,12 @@ async fn query_full_data( candidate_hash: CandidateHash, ) -> error::Result> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityRecoveryMessage, + OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let (tx, rx) = oneshot::channel(); ctx.send_message(AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx)) @@ -988,8 +1004,12 @@ impl AvailabilityRecoverySubsystem { async fn run(self, mut ctx: Context) -> SubsystemResult<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = AvailabilityRecoveryMessage, + OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let mut state = State::default(); let Self { fast_path, mut req_receiver, metrics } = self; diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index c2a17c5a42cb..87da2926774d 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -213,8 +213,12 @@ impl BitfieldDistribution { /// Start processing work as passed on from the Overseer. async fn run(self, ctx: Context) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = BitfieldDistributionMessage, + OutgoingMessages = overseer::BitfieldDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let mut state = ProtocolState::default(); let mut rng = rand::rngs::StdRng::from_entropy(); @@ -316,10 +320,7 @@ impl BitfieldDistribution { } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) -where - Context: SubsystemContext, -{ +async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) { gum::trace!(target: LOG_TARGET, ?relay_parent, ?rep, %peer, "reputation change"); ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await @@ -742,9 +743,7 @@ async fn send_tracked_gossip_message( dest: PeerId, validator: ValidatorId, message: BitfieldGossipMessage, -) where - Context: SubsystemContext, -{ +) { let job_data = if let Some(job_data) = state.per_relay_parent.get_mut(&message.relay_parent) { job_data } else { @@ -775,8 +774,12 @@ async fn send_tracked_gossip_message( impl overseer::Subsystem for BitfieldDistribution where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = BitfieldDistributionMessage, + OutgoingMessages = overseer::BitfieldDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); @@ -790,9 +793,7 @@ async fn query_basics( ctx: &mut Context, relay_parent: Hash, ) -> SubsystemResult, SigningContext)>> -where - Context: SubsystemContext, -{ +where { let (validators_tx, validators_rx) = oneshot::channel(); let (session_tx, session_rx) = oneshot::channel(); diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 860a9b0c5fa2..1cb57025e76a 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -324,8 +324,12 @@ impl Subsystem for NetworkBridge - + overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = NetworkBridgeMessage, + OutgoingMessages = overseer::NetworkBridgeOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(mut self, ctx: Context) -> SpawnedSubsystem { // The stream of networking events has to be created at initialization, otherwise the @@ -391,8 +395,12 @@ async fn handle_subsystem_messages( metrics: Metrics, ) -> Result<(), UnexpectedAbort> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = NetworkBridgeMessage, + OutgoingMessages = overseer::NetworkBridgeOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, { @@ -1039,8 +1047,12 @@ async fn run_network( where N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, - Context: SubsystemContext - + overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = NetworkBridgeMessage, + OutgoingMessages = overseer::NetworkBridgeOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let shared = Shared::default(); diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 6f23600ae8f3..7734fa64e6fb 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -370,8 +370,12 @@ async fn distribute_collation( result_sender: Option>, ) -> Result<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let relay_parent = receipt.descriptor.relay_parent; @@ -462,15 +466,19 @@ where /// Get the Id of the Core that is assigned to the para being collated on if any /// and the total number of cores. async fn determine_core( - ctx: &mut Context, + sender: &mut SubsystemSender, para_id: ParaId, relay_parent: Hash, ) -> Result> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { - let cores = get_availability_cores(ctx, relay_parent).await?; + let cores = get_availability_cores(sender, relay_parent).await?; for (idx, core) in cores.iter().enumerate() { if let CoreState::Scheduled(occupied) = core { @@ -501,8 +509,12 @@ async fn determine_our_validators( relay_parent: Hash, ) -> Result where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let session_index = runtime.get_session_index_for_child(ctx.sender(), relay_parent).await?; let info = &runtime @@ -532,8 +544,12 @@ where /// Issue a `Declare` collation message to the given `peer`. async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let declare_signature_payload = protocol_v1::declare_signature_payload(&state.local_peer_id); @@ -556,8 +572,12 @@ where /// revoke the previous connection request. async fn connect_to_validators(ctx: &mut Context, validator_ids: Vec) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { // ignore address resolution failure // will reissue a new request on new collation @@ -580,8 +600,12 @@ async fn advertise_collation( relay_parent: Hash, peer: PeerId, ) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let should_advertise = state .our_validators_groups @@ -642,8 +666,12 @@ async fn process_msg( msg: CollatorProtocolMessage, ) -> Result<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { use CollatorProtocolMessage::*; @@ -756,8 +784,12 @@ async fn handle_incoming_peer_message( msg: protocol_v1::CollatorProtocolMessage, ) -> Result<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { use protocol_v1::CollatorProtocolMessage::*; @@ -837,8 +869,12 @@ async fn handle_incoming_request( req: IncomingRequest, ) -> Result<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let _span = state .span_per_relay_parent @@ -913,8 +949,12 @@ async fn handle_peer_view_change( peer_id: PeerId, view: View, ) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let current = state.peer_views.entry(peer_id.clone()).or_default(); @@ -935,8 +975,12 @@ async fn handle_network_msg( bridge_message: NetworkBridgeEvent, ) -> Result<()> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { use NetworkBridgeEvent::*; @@ -1029,8 +1073,12 @@ pub(crate) async fn run( metrics: Metrics, ) -> std::result::Result<(), FatalError> where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { use OverseerSignal::*; diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 21a5fd9baafd..0256d1588af5 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -100,8 +100,12 @@ impl CollatorProtocolSubsystem { async fn run(self, ctx: Context) -> std::result::Result<(), error::FatalError> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => @@ -114,9 +118,14 @@ impl CollatorProtocolSubsystem { impl overseer::Subsystem for CollatorProtocolSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, - ::Sender: SubsystemSender, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, + ::Sender: + SubsystemSender, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index 8b91e8ef25a1..e4d6160255c0 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -632,8 +632,12 @@ fn collator_peer_id( async fn disconnect_peer(ctx: &mut Context, peer_id: PeerId) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { ctx.send_message(NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation)) .await @@ -646,8 +650,12 @@ async fn fetch_collation( pc: PendingCollation, id: CollatorId, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let (tx, rx) = oneshot::channel(); @@ -691,9 +699,7 @@ async fn report_collator( ctx: &mut Context, peer_data: &HashMap, id: CollatorId, -) where - Context: SubsystemContext, -{ +) { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(ctx, peer_id, COST_REPORT_BAD).await; } @@ -705,8 +711,12 @@ async fn note_good_collation( peer_data: &HashMap, id: CollatorId, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(ctx, peer_id, BENEFIT_NOTIFY_GOOD).await; @@ -720,8 +730,12 @@ async fn notify_collation_seconded( relay_parent: Hash, statement: SignedFullStatement, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let wire_message = protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement.into()); @@ -762,8 +776,12 @@ async fn request_collation( peer_id: PeerId, result: oneshot::Sender<(CandidateReceipt, PoV)>, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { if !state.view.contains(&relay_parent) { gum::debug!( @@ -829,8 +847,12 @@ async fn process_incoming_peer_message( origin: PeerId, msg: protocol_v1::CollatorProtocolMessage, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { use protocol_v1::CollatorProtocolMessage::*; use sp_runtime::traits::AppVerify; @@ -1018,8 +1040,12 @@ async fn handle_our_view_change( view: OurView, ) -> Result<()> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let old_view = std::mem::replace(&mut state.view, view); @@ -1077,8 +1103,12 @@ async fn handle_network_msg( bridge_message: NetworkBridgeEvent, ) -> Result<()> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { use NetworkBridgeEvent::*; @@ -1115,8 +1145,12 @@ async fn process_msg( msg: CollatorProtocolMessage, state: &mut State, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { use CollatorProtocolMessage::*; @@ -1218,8 +1252,12 @@ pub(crate) async fn run( metrics: Metrics, ) -> std::result::Result<(), crate::error::FatalError> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { let mut state = State { metrics, ..Default::default() }; @@ -1305,8 +1343,12 @@ async fn poll_requests( /// Dequeue another collation and fetch. async fn dequeue_next_collation_and_fetch( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, state: &mut State, relay_parent: Hash, // The collator we tried to fetch from last. @@ -1340,8 +1382,12 @@ async fn handle_collation_fetched_result( state: &mut State, (mut collation_event, res): PendingCollationFetch, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { // If no prior collation for this relay parent has been seconded, then // memorize the `collation_event` for that `relay_parent`, such that we may @@ -1406,8 +1452,12 @@ async fn disconnect_inactive_peers( eviction_policy: &crate::CollatorEvictionPolicy, peers: &HashMap, ) where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::SubsystemContext< + Message = CollatorProtocolMessage, + OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { for (peer, peer_data) in peers { if peer_data.is_inactive(&eviction_policy) { diff --git a/node/network/dispute-distribution/src/lib.rs b/node/network/dispute-distribution/src/lib.rs index 8f45cbb11b45..16e8d7767545 100644 --- a/node/network/dispute-distribution/src/lib.rs +++ b/node/network/dispute-distribution/src/lib.rs @@ -116,8 +116,12 @@ pub struct DisputeDistributionSubsystem { impl overseer::Subsystem for DisputeDistributionSubsystem where - Context: SubsystemContext - + overseer::SubsystemContext + Context: overseer::SubsystemContext< + Message = DisputeDistributionMessage, + OutgoingMessages = overseer::DisputeDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + > + Sync + Send, AD: AuthorityDiscovery + Clone, @@ -162,9 +166,12 @@ where /// Start processing work as passed on from the Overseer. async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> where - Context: SubsystemContext - + overseer::SubsystemContext - + Sync + Context: overseer::SubsystemContext< + Message = DisputeDistributionMessage, + OutgoingMessages = overseer::DisputeDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + > + Sync + Send, { let receiver = DisputesReceiver::new( @@ -245,8 +252,12 @@ enum MuxedMessage { impl MuxedMessage { async fn receive( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = DisputeDistributionMessage, + OutgoingMessages = overseer::DisputeDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, from_sender: &mut mpsc::Receiver, ) -> Self { // We are only fusing here to make `select` happy, in reality we will quit if the stream diff --git a/node/network/dispute-distribution/src/receiver/mod.rs b/node/network/dispute-distribution/src/receiver/mod.rs index c7a0a324e844..ef629e93d6e2 100644 --- a/node/network/dispute-distribution/src/receiver/mod.rs +++ b/node/network/dispute-distribution/src/receiver/mod.rs @@ -132,9 +132,10 @@ impl MuxedMessage { } } -impl DisputesReceiver +impl DisputesReceiver where AD: AuthorityDiscovery, + Sender: SubsystemSender, { /// Create a new receiver which can be `run`. pub fn new( diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index f808cc7689dd..83902942a412 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -140,8 +140,12 @@ where async fn run(mut self, mut ctx: Context) -> Self where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = GossipSupportMessage, + OutgoingMessages = overseer::GossipSupportOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn get_connectivity_check_delay() -> Delay { Delay::new(LOW_CONNECTIVITY_WARN_DELAY) @@ -178,7 +182,7 @@ where gum::trace!(target: LOG_TARGET, "active leaves signal"); let leaves = activated.into_iter().map(|a| a.hash); - if let Err(e) = self.handle_active_leaves(&mut ctx, leaves).await { + if let Err(e) = self.handle_active_leaves(ctx.sender(), leaves).await { gum::debug!(target: LOG_TARGET, error = ?e); } }, @@ -191,18 +195,13 @@ where /// 1. Determine if the current session index has changed. /// 2. If it has, determine relevant validators /// and issue a connection request. - async fn handle_active_leaves( + async fn handle_active_leaves( &mut self, - ctx: &mut Context, + sender: &mut impl overseer::SubsystemSender, leaves: impl Iterator, - ) -> Result<(), util::Error> - where - Context: SubsystemContext, - Context: overseer::SubsystemContext, - { + ) -> Result<(), util::Error> { for leaf in leaves { - let current_index = - util::request_session_index_for_child(leaf, ctx.sender()).await.await??; + let current_index = util::request_session_index_for_child(leaf, sender).await.await??; let since_failure = self.last_failure.map(|i| i.elapsed()).unwrap_or_default(); let force_request = since_failure >= BACKOFF_DURATION; let leaf_session = Some((current_index, leaf)); @@ -255,13 +254,13 @@ where // by virtue of a new session being entered. Therefore we maintain // connections to a much broader set of validators. { - let mut connections = authorities_past_present_future(ctx, leaf).await?; + let mut connections = authorities_past_present_future(sender, leaf).await?; // Remove all of our locally controlled validator indices so we don't connect to ourself. // If we control none of them, don't issue connection requests - we're outside // of the 'clique' of recent validators. if remove_all_controlled(&self.keystore, &mut connections).await != 0 { - self.issue_connection_request(ctx, connections).await; + self.issue_connection_request(sender, connections).await; } } @@ -319,12 +318,9 @@ where async fn issue_connection_request( &mut self, - ctx: &mut Context, + sender: &mut impl SubsystemSender, authorities: Vec, - ) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, - { + ) { let num = authorities.len(); let mut validator_addrs = Vec::with_capacity(authorities.len()); let mut failures = 0; @@ -347,11 +343,12 @@ where self.resolved_authorities = resolved; gum::debug!(target: LOG_TARGET, %num, "Issuing a connection request"); - ctx.send_message(NetworkBridgeMessage::ConnectToResolvedValidators { - validator_addrs, - peer_set: PeerSet::Validation, - }) - .await; + sender + .send_message(NetworkBridgeMessage::ConnectToResolvedValidators { + validator_addrs, + peer_set: PeerSet::Validation, + }) + .await; // issue another request for the same session // if at least a third of the authorities were not resolved. @@ -442,15 +439,11 @@ where } // Get the authorities of the past, present, and future. -async fn authorities_past_present_future( - ctx: &mut Context, +async fn authorities_past_present_future( + sender: &mut impl SubsystemSender, relay_parent: Hash, -) -> Result, util::Error> -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, -{ - let authorities = util::request_authorities(relay_parent, ctx.sender()).await.await??; +) -> Result, util::Error> { + let authorities = util::request_authorities(relay_parent, sender).await.await??; gum::debug!( target: LOG_TARGET, authority_count = ?authorities.len(), @@ -500,28 +493,25 @@ async fn remove_all_controlled( /// This limits the amount of gossip peers to 2 * `sqrt(len)` and ensures the diameter of 2. /// /// [web3]: https://research.web3.foundation/en/latest/polkadot/networking/3-avail-valid.html#topology -async fn update_gossip_topology( - ctx: &mut Context, +async fn update_gossip_topology( + sender: &mut impl SubsystemSender, our_index: usize, authorities: Vec, relay_parent: Hash, session_index: SessionIndex, -) -> Result<(), util::Error> -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, -{ +) -> Result<(), util::Error> { // retrieve BABE randomness let random_seed = { let (tx, rx) = oneshot::channel(); // TODO https://github.com/paritytech/polkadot/issues/5316: // get the random seed from the `SessionInfo` instead. - ctx.send_message(RuntimeApiMessage::Request( - relay_parent, - RuntimeApiRequest::CurrentBabeEpoch(tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + relay_parent, + RuntimeApiRequest::CurrentBabeEpoch(tx), + )) + .await; let randomness = rx.await??.randomness; let mut subject = [0u8; 40]; @@ -553,7 +543,7 @@ where .map(|i| (authorities[i].clone(), ValidatorIndex::from(i as u32))) .collect(); - ctx.send_message(NetworkBridgeMessage::NewGossipTopology { + sender.send_message(NetworkBridgeMessage::NewGossipTopology { session: session_index, our_neighbors_x: row_neighbors, our_neighbors_y: column_neighbors, @@ -598,8 +588,12 @@ fn matrix_neighbors( impl overseer::Subsystem for GossipSupport where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = GossipSupportMessage, + OutgoingMessages = overseer::GossipSupportOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, AD: AuthorityDiscovery + Clone, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 4eae61f567b8..0ee8e825f8cf 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -130,8 +130,12 @@ pub struct StatementDistributionSubsystem { impl overseer::Subsystem for StatementDistributionSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext< + Message = StatementDistributionMessage, + OutgoingMessages = overseer::StatementDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision @@ -632,8 +636,12 @@ enum MuxedMessage { impl MuxedMessage { async fn receive( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext< + Message = StatementDistributionMessage, + OutgoingMessages = overseer::StatementDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, ) -> MuxedMessage { @@ -894,7 +902,7 @@ async fn circulate_statement_and_dependents( gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, relay_parent: Hash, statement: SignedFullStatement, priority_peers: Vec, @@ -1009,7 +1017,7 @@ fn is_statement_large(statement: &SignedFullStatement) -> (bool, Option) async fn circulate_statement<'a>( gossip_peers: &HashSet, peers: &mut HashMap, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, relay_parent: Hash, stored: StoredStatement<'a>, mut priority_peers: Vec, @@ -1098,7 +1106,7 @@ async fn circulate_statement<'a>( async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, relay_parent: Hash, candidate_hash: CandidateHash, active_head: &ActiveHeadData, @@ -1134,7 +1142,7 @@ async fn send_statements_about( async fn send_statements( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, @@ -1164,11 +1172,7 @@ async fn send_statements( } } -async fn report_peer( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), - peer: PeerId, - rep: Rep, -) { +async fn report_peer(ctx: &mut impl overseer::SubsystemContext, peer: PeerId, rep: Rep) { ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::ReportPeer(peer, rep))) .await } @@ -1184,7 +1188,7 @@ async fn retrieve_statement_from_message<'a>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, active_head: &'a mut ActiveHeadData, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option { @@ -1276,7 +1280,7 @@ async fn launch_request( meta: StatementMetadata, peer: PeerId, req_sender: mpsc::Sender, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, metrics: &Metrics, ) -> Option { let (task, handle) = @@ -1309,7 +1313,7 @@ async fn handle_incoming_message_and_circulate<'a>( peers: &mut HashMap, active_heads: &'a mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1365,7 +1369,7 @@ async fn handle_incoming_message<'a>( peer_data: &mut PeerData, active_heads: &'a mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1561,7 +1565,7 @@ async fn update_peer_view_and_maybe_send_unlocked( peer: PeerId, gossip_peers: &HashSet, peer_data: &mut PeerData, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, active_heads: &HashMap, new_view: View, metrics: &Metrics, @@ -1602,7 +1606,7 @@ async fn handle_network_update( authorities: &mut HashMap, active_heads: &mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, req_sender: &mpsc::Sender, update: NetworkBridgeEvent, metrics: &Metrics, @@ -1714,8 +1718,12 @@ impl StatementDistributionSubsystem { async fn run( mut self, - mut ctx: (impl SubsystemContext - + overseer::SubsystemContext), + mut ctx: impl overseer::SubsystemContext< + Message = StatementDistributionMessage, + OutgoingMessages = overseer::StatementDistributionOutgoingMessages, + Signal = OverseerSignal, + Error = SubsystemError, + >, ) -> std::result::Result<(), FatalError> { let mut peers: HashMap = HashMap::new(); let mut gossip_peers: HashSet = HashSet::new(); @@ -1942,7 +1950,7 @@ impl StatementDistributionSubsystem { async fn handle_subsystem_message( &mut self, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut impl overseer::SubsystemContext, runtime: &mut RuntimeInfo, peers: &mut HashMap, gossip_peers: &mut HashSet, diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 2bd9cbadbc46..e46f9e5e2850 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -49,10 +49,7 @@ struct Subsystem1; impl Subsystem1 { async fn run(mut ctx: Ctx) -> () where - Ctx: overseer::SubsystemContext< - Message = CandidateBackingMessage, - Signal = OverseerSignal - >, + Ctx: overseer::SubsystemContext, ::OutgoingMessages: From, { 'louy: loop { @@ -95,7 +92,7 @@ where Context: overseer::SubsystemContext< Message = CandidateBackingMessage, OutgoingMessages = overseer::CandidateBackingOutgoingMessages, - Signal = OverseerSignal + Signal = OverseerSignal, >, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index cd86a1d64668..45f8a1cf3271 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -259,16 +259,14 @@ pub(crate) fn impl_subsystem_sender( // 2. overseer-global-`AllMessages`-type let wrapped = |outgoing_wrapper: &TokenStream| { quote! { - /// `M` references the type _consumed_ by the subsystem. #[#support_crate ::async_trait] - impl SubsystemSender< #outgoing_wrapper > for #subsystem_sender_name < #outgoing_wrapper > + impl SubsystemSender< OutgoingMessage > for #subsystem_sender_name < #outgoing_wrapper > where + OutgoingMessage: Send + 'static, + #outgoing_wrapper: ::std::convert::From + Send, #all_messages_wrapper: ::std::convert::From< #outgoing_wrapper > + Send, { - async fn send_message(&mut self, msg: OutgoingMessage) - where - OutgoingMessage: Send, - #outgoing_wrapper: ::std::convert::From + Send, + async fn send_message(&mut self, msg: OutgoingMessage) { self.channels.send_and_log_error( self.signals_received.load(), @@ -278,10 +276,8 @@ pub(crate) fn impl_subsystem_sender( ).await; } - async fn send_messages(&mut self, msgs: I) + async fn send_messages(&mut self, msgs: I) where - OutgoingMessage: Send, - #outgoing_wrapper: ::std::convert::From + Send, I: IntoIterator + Send, I::IntoIter: Iterator + Send, { @@ -290,10 +286,7 @@ pub(crate) fn impl_subsystem_sender( } } - fn send_unbounded_message(&mut self, msg: OutgoingMessage) - where - OutgoingMessage: Send, - #outgoing_wrapper: ::std::convert::From + Send, + fn send_unbounded_message(&mut self, msg: OutgoingMessage) { self.channels.send_unbounded_and_log_error( self.signals_received.load(), @@ -307,10 +300,14 @@ pub(crate) fn impl_subsystem_sender( }; for outgoing_wrapper in outgoing_wrappers { - ts.extend(wrapped("e! { #outgoing_wrapper })); + ts.extend(wrapped("e! { + #outgoing_wrapper + })); } - ts.extend(wrapped("e! { () })); + ts.extend(wrapped("e! { + () + })); ts } diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 1b477519103d..b4c4d5efa2db 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -372,7 +372,7 @@ pub trait SubsystemContext: Send + 'static { // type AllMessages: From + From + std::fmt::Debug + Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: SubsystemSender + Send + 'static; + type Sender: Clone + SubsystemSender + Send + 'static; /// The error type. type Error: ::std::error::Error + ::std::convert::From + Sync + Send + 'static; @@ -405,7 +405,7 @@ pub trait SubsystemContext: Send + 'static { Self::OutgoingMessages: From + Send, T: Send, { - self.sender().send_message::(msg).await + self.sender().send_message(::from(msg)).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. @@ -416,7 +416,9 @@ pub trait SubsystemContext: Send + 'static { I::IntoIter: Send, T: Send, { - self.sender().send_messages::(msgs.into_iter()).await + self.sender() + .send_messages(msgs.into_iter().map(::from)) + .await } /// Send a message using the unbounded connection. @@ -425,8 +427,7 @@ pub trait SubsystemContext: Send + 'static { Self::OutgoingMessages: From + Send, X: Send, { - self.sender() - .send_unbounded_message::(Self::OutgoingMessages::from(msg)) + self.sender().send_unbounded_message(::from(msg)) } /// Obtain the sender. @@ -452,18 +453,16 @@ where /// Sender end of a channel to interface with a subsystem. #[async_trait::async_trait] -pub trait SubsystemSender: Send + 'static { +pub trait SubsystemSender: Clone + Send + 'static +where + OutgoingMessage: Send, +{ /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: OutgoingMessage) - where - OutgoingMessages: From, - OutgoingMessage: Send; + async fn send_message(&mut self, msg: OutgoingMessage); /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: I) + async fn send_messages(&mut self, msgs: I) where - OutgoingMessages: From, - OutgoingMessage: Send, I: IntoIterator + Send, I::IntoIter: Send; @@ -472,10 +471,7 @@ pub trait SubsystemSender: Send + 'static { /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: OutgoingMessage) - where - OutgoingMessages: From, - OutgoingMessage: Send; + fn send_unbounded_message(&mut self, msg: OutgoingMessage); } /// A future that wraps another future with a `Delay` allowing for time-limited futures. diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index cd1812f8be79..2cad3d655979 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -457,7 +457,13 @@ pub struct Overseer { #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage, sends: [])] availability_store: AvailabilityStore, - #[subsystem(no_dispatch, NetworkBridgeMessage, sends: [])] + #[subsystem(no_dispatch, NetworkBridgeMessage, sends: [ + ApprovalDistributionMessage, + StatementDistributionMessage, + BitfieldDistributionMessage, + GossipSupportMessage, + DisputeDistributionMessage, + ])] network_bridge: NetworkBridge, #[subsystem(no_dispatch, blocking, ChainApiMessage, sends: [])] @@ -475,10 +481,16 @@ pub struct Overseer { #[subsystem(no_dispatch, blocking, ApprovalVotingMessage, sends: [])] approval_voting: ApprovalVoting, - #[subsystem(GossipSupportMessage, sends: [])] + #[subsystem(GossipSupportMessage, sends: [ + NetworkBridgeMessage, + RuntimeApiMessage, + ])] gossip_support: GossipSupport, - #[subsystem(no_dispatch, blocking, DisputeCoordinatorMessage, sends: [])] + #[subsystem(no_dispatch, blocking, DisputeCoordinatorMessage, sends: [ + RuntimeApiMessage, + ChainApiMessage, + ])] dispute_coordinator: DisputeCoordinator, #[subsystem(no_dispatch, DisputeDistributionMessage, sends: [])] diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index 06249e109d06..d7d1b6d2ff39 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -158,10 +158,10 @@ where self.tx.send(msg.into()).await.expect("test overseer no longer live"); } - async fn send_messages(&mut self, msgs: X) + async fn send_messages(&mut self, msgs: I) where - X: IntoIterator + Send, - X::IntoIter: Send, + I: IntoIterator + Send, + I::IntoIter: Send, { let mut iter = stream::iter(msgs.into_iter().map(|msg| Ok(msg.into()))); self.tx.send_all(&mut iter).await.expect("test overseer no longer live"); @@ -180,16 +180,16 @@ pub struct TestSubsystemContext { } #[async_trait::async_trait] -impl overseer::SubsystemContext for TestSubsystemContext +impl overseer::SubsystemContext for TestSubsystemContext where M: std::fmt::Debug + Send + 'static, AllMessages: From, - S: SpawnNamed + Send + 'static, + Spawner: SpawnNamed + Send + 'static, { type Message = M; type Sender = TestSubsystemSender; type Signal = OverseerSignal; - type AllMessages = AllMessages; + type OutgoingMessages = AllMessages; type Error = SubsystemError; async fn try_recv(&mut self) -> Result>, ()> { @@ -317,7 +317,9 @@ pub struct ForwardSubsystem(pub mpsc::Sender); impl overseer::Subsystem for ForwardSubsystem where M: std::fmt::Debug + Send + 'static, - Context: SubsystemContext + overseer::SubsystemContext, + Context: + overseer::SubsystemContext, + ::OutgoingMessages: From + Send, { fn start(mut self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index 3c862b1fc704..0a330d7f04fc 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -17,7 +17,10 @@ //! A utility for fetching all unknown blocks based on a new chain-head hash. use futures::{channel::oneshot, prelude::*}; -use polkadot_node_subsystem::{messages::ChainApiMessage, SubsystemSender}; +use polkadot_node_subsystem::{ + messages::{ChainApiMessage, RuntimeApiMessage}, + SubsystemSender, +}; use polkadot_primitives::v2::{BlockNumber, Hash, Header}; /// Given a new chain-head hash, this determines the hashes of all new blocks we should track @@ -39,7 +42,7 @@ pub async fn determine_new_blocks( lower_bound_number: BlockNumber, ) -> Result, E> where - Sender: SubsystemSender, + Sender: SubsystemSender, { const ANCESTRY_STEP: usize = 4; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index b249fa11f0eb..63c835e6f9c9 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -144,7 +144,7 @@ pub async fn request_from_runtime( ) -> RuntimeApiReceiver where RequestBuilder: FnOnce(RuntimeApiSender) -> RuntimeApiRequest, - Sender: SubsystemSender, + Sender: SubsystemSender, { let (tx, rx) = oneshot::channel(); @@ -176,7 +176,7 @@ macro_rules! specialize_requests { $( $param_name: $param_ty, )* - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::SubsystemSender, ) -> RuntimeApiReceiver<$return_ty> { request_from_runtime(parent, sender, |tx| RuntimeApiRequest::$request_variant( @@ -329,11 +329,14 @@ pub struct Validator { impl Validator { /// Get a struct representing this node's validator if this node is in fact a validator in the context of the given block. - pub async fn new( + pub async fn new( parent: Hash, keystore: SyncCryptoStorePtr, - sender: &mut impl SubsystemSender, - ) -> Result { + sender: &mut S, + ) -> Result + where + S: SubsystemSender, + { // Note: request_validators and request_session_index_for_child do not and cannot // run concurrently: they both have a mutable handle to the same sender. // However, each of them returns a oneshot::Receiver, and those are resolved concurrently. @@ -418,49 +421,25 @@ pub enum FromJobCommand { } /// A sender for messages from jobs, as well as commands to the overseer. -pub struct JobSender { +pub struct JobSender { sender: S, from_job: mpsc::Sender, } // A custom clone impl, since M does not need to impl `Clone` // which `#[derive(Clone)]` requires. -impl Clone for JobSender { +impl Clone for JobSender { fn clone(&self) -> Self { Self { sender: self.sender.clone(), from_job: self.from_job.clone() } } } -impl JobSender { +impl JobSender { /// Get access to the underlying subsystem sender. pub fn subsystem_sender(&mut self) -> &mut S { &mut self.sender } - /// Send a direct message to some other `Subsystem`, routed based on message type. - pub async fn send_message(&mut self, msg: impl Into) { - self.sender.send_message(msg.into()).await - } - - /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - pub async fn send_messages(&mut self, msgs: T) - where - T: IntoIterator + Send, - T::IntoIter: Send, - M: Into, - { - self.sender.send_messages(msgs.into_iter().map(|m| m.into())).await - } - - /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message - /// type. - /// - /// This function should be used only when there is some other bounding factor on the messages - /// sent with it. Otherwise, it risks a memory leak. - pub fn send_unbounded_message(&mut self, msg: impl Into) { - self.sender.send_unbounded_message(msg.into()) - } - /// Send a command to the subsystem, to be relayed onwards to the overseer. pub async fn send_command(&mut self, msg: FromJobCommand) -> Result<(), mpsc::SendError> { self.from_job.send(msg).await @@ -470,23 +449,23 @@ impl JobSender { #[async_trait::async_trait] impl overseer::SubsystemSender for JobSender where - M: Send + 'static + Into, - S: SubsystemSender + Clone, + M: Send + 'static, + S: SubsystemSender + Clone, { async fn send_message(&mut self, msg: M) { - self.sender.send_message(msg.into()).await + self.sender.send_message(msg).await } - async fn send_messages(&mut self, msgs: T) + async fn send_messages(&mut self, msgs: I) where - T: IntoIterator + Send, - T::IntoIter: Send, + I: IntoIterator + Send, + I::IntoIter: Send, { - self.sender.send_messages(msgs.into_iter().map(|m| m.into())).await + self.sender.send_messages(msgs).await } fn send_unbounded_message(&mut self, msg: M) { - self.sender.send_unbounded_message(msg.into()) + self.sender.send_unbounded_message(msg) } } @@ -506,6 +485,9 @@ impl fmt::Debug for FromJobCommand { pub trait JobTrait: Unpin + Sized { /// Message type used to send messages to the job. type ToJob: 'static + BoundToRelayParent + Send; + + type OutgoingMessages: 'static + Send; + /// Job runtime error. type Error: 'static + std::error::Error + Send; /// Extra arguments this job needs to run properly. @@ -525,13 +507,15 @@ pub trait JobTrait: Unpin + Sized { /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run( + fn run( leaf: ActivatedLeaf, run_args: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, sender: JobSender, - ) -> Pin> + Send>>; + ) -> Pin> + Send>> + where + S: SubsystemSender<::OutgoingMessages>; } /// Error which can be returned by the jobs manager @@ -580,7 +564,7 @@ where sender: Sender, ) where Job: JobTrait, - Sender: SubsystemSender, + Sender: SubsystemSender<::OutgoingMessages>, { let hash = leaf.hash; let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -697,8 +681,12 @@ impl JobSubsystem { pub async fn run(self, mut ctx: Context) where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, - Context: SubsystemContext::ToJob, Signal = OverseerSignal>, - ::Sender: SubsystemSender, + Context: SubsystemContext< + Message = ::ToJob, + OutgoingMessages = ::OutgoingMessages, + Signal = OverseerSignal, + >, + ::Sender: SubsystemSender<::OutgoingMessages>, Job: 'static + JobTrait + Send, ::RunArgs: Clone + Sync, ::ToJob: @@ -773,11 +761,14 @@ impl JobSubsystem { impl Subsystem for JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, - Context: SubsystemContext, + Context: SubsystemContext< + Message = Job::ToJob, + Signal = OverseerSignal, + OutgoingMessages = ::OutgoingMessages, + >, Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - ::ToJob: - Sync + From<::Message>, + ::ToJob: Sync + From<::Message>, Job::Metrics: Sync, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index c8a467d47d38..f4e782c8389a 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -95,15 +95,15 @@ pub struct RollingSessionWindow { impl RollingSessionWindow { /// Initialize a new session info cache with the given window size. pub async fn new( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + mut sender: impl overseer::SubsystemSender, window_size: SessionWindowSize, block_hash: Hash, ) -> Result { - let session_index = get_session_index_for_child(ctx, block_hash).await?; + let session_index = get_session_index_for_child(&mut sender, block_hash).await?; let window_start = session_index.saturating_sub(window_size.get() - 1); - match load_all_sessions(ctx, block_hash, window_start, session_index).await { + match load_all_sessions(&mut sender, block_hash, window_start, session_index).await { Err(kind) => Err(SessionsUnavailable { kind, info: Some(SessionsUnavailableInfo { @@ -154,10 +154,10 @@ impl RollingSessionWindow { /// some backwards drift in session index is acceptable. pub async fn cache_session_info_for_head( &mut self, - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + sender: &mut impl overseer::SubsystemSender, block_hash: Hash, ) -> Result { - let session_index = get_session_index_for_child(ctx, block_hash).await?; + let session_index = get_session_index_for_child(sender, block_hash).await?; let old_window_start = self.earliest_session; @@ -177,7 +177,7 @@ impl RollingSessionWindow { let fresh_start = if latest < window_start { window_start } else { latest + 1 }; - match load_all_sessions(ctx, block_hash, fresh_start, session_index).await { + match load_all_sessions(sender, block_hash, fresh_start, session_index).await { Err(kind) => Err(SessionsUnavailable { kind, info: Some(SessionsUnavailableInfo { @@ -215,17 +215,18 @@ impl RollingSessionWindow { // cleaner to just call the runtime API directly without needing to create an instance // of `RuntimeInfo`. async fn get_session_index_for_child( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + sender: &mut impl overseer::SubsystemSender, block_hash: Hash, ) -> Result { let (s_tx, s_rx) = oneshot::channel(); // We're requesting session index of a child to populate the cache in advance. - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::SessionIndexForChild(s_tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::SessionIndexForChild(s_tx), + )) + .await; match s_rx.await { Ok(Ok(s)) => Ok(s), @@ -243,7 +244,7 @@ async fn get_session_index_for_child( } async fn load_all_sessions( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + sender: &mut impl overseer::SubsystemSender, block_hash: Hash, start: SessionIndex, end_inclusive: SessionIndex, @@ -251,11 +252,12 @@ async fn load_all_sessions( let mut v = Vec::new(); for i in start..=end_inclusive { let (tx, rx) = oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::SessionInfo(i, tx), - )) - .await; + sender + .send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::SessionInfo(i, tx), + )) + .await; let session_info = match rx.await { Ok(Ok(Some(s))) => s, @@ -319,13 +321,16 @@ mod tests { let hash = header.hash(); + let mut sender = ctx.sender(); + let test_fut = { Box::pin(async move { let window = match window { - None => - RollingSessionWindow::new(&mut ctx, TEST_WINDOW_SIZE, hash).await.unwrap(), + None => RollingSessionWindow::new(&mut sender, TEST_WINDOW_SIZE, hash) + .await + .unwrap(), Some(mut window) => { - window.cache_session_info_for_head(&mut ctx, hash).await.unwrap(); + window.cache_session_info_for_head(&mut sender, hash).await.unwrap(); window }, }; diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 4286eba89a54..14d4909e3d47 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -25,7 +25,9 @@ use sp_application_crypto::AppKey; use sp_core::crypto::ByteArray; use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; -use polkadot_node_subsystem::{SubsystemContext, SubsystemSender}; +use polkadot_node_subsystem::{ + messages::RuntimeApiMessage, overseer, SubsystemContext, SubsystemSender, +}; use polkadot_primitives::v2::{ CandidateEvent, CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, ScrapedOnChainVotes, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, @@ -123,7 +125,7 @@ impl RuntimeInfo { parent: Hash, ) -> Result where - Sender: SubsystemSender, + Sender: SubsystemSender, { match self.session_index_cache.get(&parent) { Some(index) => Ok(*index), @@ -143,7 +145,7 @@ impl RuntimeInfo { relay_parent: Hash, ) -> Result<&'a ExtendedSessionInfo> where - Sender: SubsystemSender, + Sender: SubsystemSender, { let session_index = self.get_session_index_for_child(sender, relay_parent).await?; @@ -161,7 +163,7 @@ impl RuntimeInfo { session_index: SessionIndex, ) -> Result<&'a ExtendedSessionInfo> where - Sender: SubsystemSender, + Sender: SubsystemSender, { if !self.session_info_cache.contains(&session_index) { let session_info = @@ -190,7 +192,7 @@ impl RuntimeInfo { std::result::Result, UncheckedSigned>, > where - Sender: SubsystemSender, + Sender: SubsystemSender, Payload: EncodeAs + Clone, RealPayload: Encode + Clone, { @@ -257,25 +259,25 @@ where } /// Request availability cores from the runtime. -pub async fn get_availability_cores( - ctx: &mut Context, +pub async fn get_availability_cores( + sender: &mut Sender, relay_parent: Hash, ) -> Result> where - Context: SubsystemContext, + Sender: overseer::SubsystemSender, { - recv_runtime(request_availability_cores(relay_parent, ctx.sender()).await).await + recv_runtime(request_availability_cores(relay_parent, sender).await).await } /// Variant of `request_availability_cores` that only returns occupied ones. -pub async fn get_occupied_cores( - ctx: &mut Context, +pub async fn get_occupied_cores( + sender: &mut Sender, relay_parent: Hash, ) -> Result> where - Context: SubsystemContext, + Sender: overseer::SubsystemSender, { - let cores = get_availability_cores(ctx, relay_parent).await?; + let cores = get_availability_cores(sender, relay_parent).await?; Ok(cores .into_iter() @@ -290,17 +292,16 @@ where } /// Get group rotation info based on the given `relay_parent`. -pub async fn get_group_rotation_info( - ctx: &mut Context, +pub async fn get_group_rotation_info( + sender: &mut Sender, relay_parent: Hash, ) -> Result where - Context: SubsystemContext, + Sender: overseer::SubsystemSender, { // We drop `groups` here as we don't need them, because of `RuntimeInfo`. Ideally we would not // fetch them in the first place. - let (_, info) = - recv_runtime(request_validator_groups(relay_parent, ctx.sender()).await).await?; + let (_, info) = recv_runtime(request_validator_groups(relay_parent, sender).await).await?; Ok(info) } @@ -310,7 +311,7 @@ pub async fn get_candidate_events( relay_parent: Hash, ) -> Result> where - Sender: SubsystemSender, + Sender: SubsystemSender, { recv_runtime(request_candidate_events(relay_parent, sender).await).await } @@ -333,7 +334,7 @@ pub async fn get_validation_code_by_hash( validation_code_hash: ValidationCodeHash, ) -> Result> where - Sender: SubsystemSender, + Sender: SubsystemSender, { recv_runtime(request_validation_code_by_hash(relay_parent, validation_code_hash, sender).await) .await diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 166b4d557508..d4b9b547008c 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -59,6 +59,7 @@ enum Error { impl JobTrait for FakeCollatorProtocolJob { type ToJob = CollatorProtocolMessage; + type OutgoingMessages = CollatorProtocolOutgoingMessages; type Error = Error; type RunArgs = bool; type Metrics = (); diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 054f0d5997bb..2ffd4871f4f8 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -24,9 +24,7 @@ pub use jaeger::*; pub use polkadot_node_jaeger as jaeger; -pub use polkadot_overseer::{ - self as overseer, ActiveLeavesUpdate, OverseerConnector, OverseerSignal, -}; +pub use polkadot_overseer::{self as overseer, *}; pub use polkadot_node_subsystem_types::{ errors::{self, *}, @@ -54,37 +52,5 @@ pub type FromOverseer = polkadot_overseer::gen::FromOverseer = polkadot_overseer::gen::SubsystemInstance; -/// Sender trait for the `AllMessages` wrapper. -pub trait SubsystemSender: polkadot_overseer::gen::SubsystemSender {} - -impl SubsystemSender for T where T: polkadot_overseer::gen::SubsystemSender -{} - /// Spawned subsystem. pub type SpawnedSubsystem = polkadot_overseer::gen::SpawnedSubsystem; - -/// Convenience trait specialization. -pub trait SubsystemContext: - polkadot_overseer::gen::SubsystemContext< - Signal = OverseerSignal, - AllMessages = messages::AllMessages, - Error = SubsystemError, -> -{ - /// The message type the subsystem consumes. - type Message: std::fmt::Debug + Send + 'static; - /// Sender type to communicate with other subsystems. - type Sender: SubsystemSender + Send + Clone + 'static; -} - -impl SubsystemContext for T -where - T: polkadot_overseer::gen::SubsystemContext< - Signal = OverseerSignal, - AllMessages = messages::AllMessages, - Error = SubsystemError, - >, -{ - type Message = ::Message; - type Sender = ::Sender; -} From b91630e2b6603b38c9893e621982bcb159bbdcf2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 16:01:24 +0200 Subject: [PATCH 013/139] fixup pvf-checker --- node/core/pvf-checker/src/lib.rs | 29 +++++++++++------------- node/core/pvf-checker/src/runtime_api.rs | 10 ++++---- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index e0cfde5e7b56..80c4a0d4cd0a 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -22,8 +22,8 @@ use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered}; use polkadot_node_subsystem::{ - messages::{CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage}, - overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, + messages::{CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage, RuntimeApiMessage}, + overseer::{self, PvfCheckerOutgoingMessages}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, SubsystemResult, SubsystemSender, }; use polkadot_primitives::v2::{ @@ -62,12 +62,7 @@ impl PvfCheckerSubsystem { impl overseer::Subsystem for PvfCheckerSubsystem where - Context: overseer::SubsystemContext< - Message = PvfCheckerMessage, - OutgoingMessages = overseer::PvfCheckerOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::PvfCheckerContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { if self.enabled { @@ -138,7 +133,9 @@ where OutgoingMessages = overseer::PvfCheckerOutgoingMessages, Signal = OverseerSignal, Error = SubsystemError, + // Sender is ommitted here >, + ::Sender: overseer::PvfCheckerSenderTrait, { let mut state = State { credentials: None, @@ -187,7 +184,7 @@ where /// Handle an incoming PVF pre-check result from the candidate-validation subsystem. async fn handle_pvf_check( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::PvfCheckerSenderTrait, keystore: &SyncCryptoStorePtr, metrics: &Metrics, outcome: PreCheckOutcome, @@ -255,7 +252,7 @@ struct Conclude; async fn handle_from_overseer( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::PvfCheckerSenderTrait, keystore: &SyncCryptoStorePtr, metrics: &Metrics, from_overseer: FromOverseer, @@ -281,7 +278,7 @@ async fn handle_from_overseer( async fn handle_leaves_update( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::PvfCheckerSenderTrait, keystore: &SyncCryptoStorePtr, metrics: &Metrics, update: ActiveLeavesUpdate, @@ -363,7 +360,7 @@ struct ActivationEffect { /// Returns `None` if the PVF pre-checking runtime API is not supported for the given leaf hash. async fn examine_activation( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::PvfCheckerSenderTrait, keystore: &SyncCryptoStorePtr, leaf_hash: Hash, leaf_number: BlockNumber, @@ -422,7 +419,7 @@ async fn examine_activation( /// Checks the active validators for the given leaf. If we have a signing key for one of them, /// returns the [`SigningCredentials`]. async fn check_signing_credentials( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, leaf: Hash, ) -> Option { @@ -451,7 +448,7 @@ async fn check_signing_credentials( /// /// If the validator already voted for the given code, this function does nothing. async fn sign_and_submit_pvf_check_statement( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::PvfCheckerSenderTrait, keystore: &SyncCryptoStorePtr, voted: &mut HashSet, credentials: &SigningCredentials, @@ -543,7 +540,7 @@ async fn sign_and_submit_pvf_check_statement( /// into the `currently_checking` set. async fn initiate_precheck( state: &mut State, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::PvfCheckerSenderTrait, relay_parent: Hash, validation_code_hash: ValidationCodeHash, metrics: &Metrics, @@ -553,7 +550,7 @@ async fn initiate_precheck( let (tx, rx) = oneshot::channel(); sender .send_message( - CandidateValidationMessage::PreCheck(relay_parent, validation_code_hash, tx).into(), + CandidateValidationMessage::PreCheck(relay_parent, validation_code_hash, tx) ) .await; diff --git a/node/core/pvf-checker/src/runtime_api.rs b/node/core/pvf-checker/src/runtime_api.rs index 798fb3f1abfb..ccdcce4c2c83 100644 --- a/node/core/pvf-checker/src/runtime_api.rs +++ b/node/core/pvf-checker/src/runtime_api.rs @@ -26,7 +26,7 @@ use polkadot_primitives::v2::{ }; pub(crate) async fn session_index_for_child( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, relay_parent: Hash, ) -> Result { let (tx, rx) = oneshot::channel(); @@ -34,7 +34,7 @@ pub(crate) async fn session_index_for_child( } pub(crate) async fn validators( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, relay_parent: Hash, ) -> Result, RuntimeRequestError> { let (tx, rx) = oneshot::channel(); @@ -42,7 +42,7 @@ pub(crate) async fn validators( } pub(crate) async fn submit_pvf_check_statement( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, relay_parent: Hash, stmt: PvfCheckStatement, signature: ValidatorSignature, @@ -58,7 +58,7 @@ pub(crate) async fn submit_pvf_check_statement( } pub(crate) async fn pvfs_require_precheck( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, relay_parent: Hash, ) -> Result, RuntimeRequestError> { let (tx, rx) = oneshot::channel(); @@ -73,7 +73,7 @@ pub(crate) enum RuntimeRequestError { } pub(crate) async fn runtime_api_request( - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, relay_parent: Hash, request: RuntimeApiRequest, receiver: oneshot::Receiver>, From ebadddfbc26316eb440d85ec2cf2f1de2133df6c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 16:01:41 +0200 Subject: [PATCH 014/139] more collector traits --- node/overseer/overseer-gen/examples/dummy.rs | 14 +- .../proc-macro/src/impl_subsystem.rs | 221 +++++++++++++++++- node/overseer/src/lib.rs | 2 +- 3 files changed, 227 insertions(+), 10 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 31b1c4caccf8..aedb3feda676 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -8,8 +8,11 @@ use std::collections::HashMap; #[derive(Default)] pub struct AwesomeSubSys; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { - fn start(self, mut ctx: XxxSubsystemContext) -> SpawnedSubsystem { +impl ::polkadot_overseer_gen::Subsystem for AwesomeSubSys +where + Ctx: AwesomeSubSysContextTrait, +{ + fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( "AwesomeSubsys", @@ -25,8 +28,11 @@ impl ::polkadot_overseer_gen::Subsystem, Yikes> f #[derive(Default)] pub struct GoblinTower; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { - fn start(self, mut ctx: XxxSubsystemContext) -> SpawnedSubsystem { +impl ::polkadot_overseer_gen::Subsystem for GoblinTower +where + Ctx: GoblinTowerContextTrait, +{ + fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( "GoblinTower", diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 45f8a1cf3271..5e4aa008fdad 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -142,8 +142,8 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { } } - let subsystem_sender_name = Ident::new(&(overseer_name.to_string() + "Sender"), span); - let subsystem_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), span); + let subsystem_sender_name = &Ident::new(&(overseer_name.to_string() + "Sender"), span); + let subsystem_ctx_name = &Ident::new(&(overseer_name.to_string() + "SubsystemContext"), span); ts.extend(impl_subsystem_context(info, &subsystem_sender_name, &subsystem_ctx_name)); ts.extend(impl_associate_outgoing_messages_trait(&all_messages_wrapper)); @@ -162,7 +162,19 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { // Create all subsystem specific types, one by one for ssf in info.subsystems() { let subsystem_name = ssf.generic.to_string(); - let outgoing_wrapper = Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span); + let outgoing_wrapper = &Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span); + + let subsystem_ctx_trait= &Ident::new(&(subsystem_name.clone() + "ContextTrait"), span); + let subsystem_sender_trait= &Ident::new(&(subsystem_name.clone() + "SenderTrait"), span); + + ts.extend(impl_per_subsystem_trait( + info, + subsystem_ctx_trait, + subsystem_sender_name, + subsystem_sender_trait, + &ssf.message_to_consume, + &ssf.messages_to_send, + outgoing_wrapper)); ts.extend(impl_associate_outgoing_messages(&ssf.message_to_consume, &outgoing_wrapper)); @@ -346,6 +358,201 @@ pub(crate) fn impl_associate_outgoing_messages( } } + +pub(crate) fn impl_per_subsystem_trait( + info: &OverseerInfo, + subsystem_ctx_trait: &Ident, + subsystem_sender_name: &Ident, + subsystem_sender_trait: &Ident, + consumes: &Path, + outgoing: &[Path], + outgoing_wrapper: &Ident, +) -> TokenStream { + let all_messages_wrapper = &info.message_wrapper; + let signal = &info.extern_signal_ty; + let error_ty = &info.extern_error_ty; + let support_crate = info.support_crate_name(); + + let mut ts = TokenStream::new(); + + let acc_sender_trait_bounds = quote !{ + #support_crate ::SubsystemSender< #outgoing_wrapper > + #( + + #support_crate ::SubsystemSender< #outgoing > + )* + }; + + ts.extend(quote!{ + /// A abstracting trait for usage with subsystems. + pub trait #subsystem_sender_trait : #acc_sender_trait_bounds + {} + + impl #subsystem_sender_trait for T + where + T: #acc_sender_trait_bounds + {} + }); + + + + let where_clause = quote !{ + #consumes: AssociateOutgoing, + // : <#consumes as AssociateOutgoing>::OutgoingMessages, + #all_messages_wrapper: From< #consumes >, + #subsystem_sender_name < #outgoing_wrapper >: #subsystem_sender_trait + #acc_sender_trait_bounds, + }; + + ts.extend(quote!{ + /// Accumuative trait for a particular subsystem wrapper. + pub trait #subsystem_ctx_trait : SubsystemContext < + Message = #consumes, + Signal = #signal, + OutgoingMessages = #outgoing_wrapper, + Sender = #subsystem_sender_name < #outgoing_wrapper > , + Error = #error_ty, + > + where + #where_clause + {} + + impl #subsystem_ctx_trait for T + where + T: SubsystemContext < + Message = #consumes, + Signal = #signal, + OutgoingMessages = #outgoing_wrapper, + Sender = #subsystem_sender_name < #outgoing_wrapper >, + Error = #error_ty, + >, + #where_clause + {} + }); + + + // // impl the subsystem context trait (alt!) + // ts.extend(quote!{ + + // #[#support_crate ::async_trait] + // impl #support_crate ::SubsystemContext for #subsystem_ctx_name <#consumes> + // where + // #consumes: AssociateOutgoing, + // // : <#consumes as AssociateOutgoing>::OutgoingMessages, + // #all_messages_wrapper: From< #consumes >, + // #subsystem_sender_name: + // #support_crate ::SubsystemSender< #outgoing_wrapper > + // #( + // + #support_crate ::SubsystemSender< #outgoing > + // )* + // , + // { + // type Message = #consumes; + // type Signal = #signal; + // type OutgoingMessages = #outgoing_wrapper; + // // type AllMessages = #all_messages_wrapper; + // type Sender = #subsystem_sender_name; + // type Error = #error_ty; + + // async fn try_recv(&mut self) -> ::std::result::Result>, ()> { + // match #support_crate ::poll!(self.recv()) { + // #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), + // #support_crate ::Poll::Pending => Ok(None), + // } + // } + + // async fn recv(&mut self) -> ::std::result::Result, Self::Error> { + // loop { + // // If we have a message pending an overseer signal, we only poll for signals + // // in the meantime. + // if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { + // if needs_signals_received <= self.signals_received.load() { + // return Ok( #support_crate ::FromOverseer::Communication { msg }); + // } else { + // self.pending_incoming = Some((needs_signals_received, msg)); + + // // wait for next signal. + // let signal = self.signals.next().await + // .ok_or(#support_crate ::OverseerError::Context( + // "Signal channel is terminated and empty." + // .to_owned() + // ))?; + + // self.signals_received.inc(); + // return Ok( #support_crate ::FromOverseer::Signal(signal)) + // } + // } + + // let mut await_message = self.messages.next().fuse(); + // let mut await_signal = self.signals.next().fuse(); + // let signals_received = self.signals_received.load(); + // let pending_incoming = &mut self.pending_incoming; + + // // Otherwise, wait for the next signal or incoming message. + // let from_overseer = #support_crate ::futures::select_biased! { + // signal = await_signal => { + // let signal = signal + // .ok_or( #support_crate ::OverseerError::Context( + // "Signal channel is terminated and empty." + // .to_owned() + // ))?; + + // #support_crate ::FromOverseer::Signal(signal) + // } + // msg = await_message => { + // let packet = msg + // .ok_or( #support_crate ::OverseerError::Context( + // "Message channel is terminated and empty." + // .to_owned() + // ))?; + + // if packet.signals_received > signals_received { + // // wait until we've received enough signals to return this message. + // *pending_incoming = Some((packet.signals_received, packet.message)); + // continue; + // } else { + // // we know enough to return this message. + // #support_crate ::FromOverseer::Communication { msg: packet.message} + // } + // } + // }; + + // if let #support_crate ::FromOverseer::Signal(_) = from_overseer { + // self.signals_received.inc(); + // } + + // return Ok(from_overseer); + // } + // } + + // fn sender(&mut self) -> &mut Self::Sender { + // &mut self.to_subsystems + // } + + // fn spawn(&mut self, name: &'static str, s: Pin + Send>>) + // -> ::std::result::Result<(), #error_ty> + // { + // self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { + // name, + // subsystem: Some(self.name()), + // s, + // }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; + // Ok(()) + // } + + // fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + // -> ::std::result::Result<(), #error_ty> + // { + // self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { + // name, + // subsystem: Some(self.name()), + // s, + // }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; + // Ok(()) + // } + // } + // }); + ts +} + /// Implement a builder pattern for the `Overseer`-type, /// which acts as the gateway to constructing the overseer. pub(crate) fn impl_subsystem_context( @@ -358,7 +565,7 @@ pub(crate) fn impl_subsystem_context( let error_ty = &info.extern_error_ty; let support_crate = info.support_crate_name(); - let ts = quote! { + let mut ts = quote! { /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or to spawn it's [`SubsystemJob`]s. @@ -412,6 +619,10 @@ pub(crate) fn impl_subsystem_context( self.name } } + }; + + // impl the subsystem context trait + ts.extend(quote!{ #[#support_crate ::async_trait] impl #support_crate ::SubsystemContext for #subsystem_ctx_name @@ -529,7 +740,7 @@ pub(crate) fn impl_subsystem_context( Ok(()) } } - }; + }); ts } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 2cad3d655979..9ff952e41877 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -421,7 +421,7 @@ pub struct Overseer { #[subsystem(no_dispatch, CandidateValidationMessage, sends: [])] candidate_validation: CandidateValidation, - #[subsystem(no_dispatch, PvfCheckerMessage, sends: [])] + #[subsystem(no_dispatch, PvfCheckerMessage, sends: [CandidateValidationMessage, RuntimeApiMessage])] pvf_checker: PvfChecker, #[subsystem(no_dispatch, CandidateBackingMessage, sends: [ From 92e4219fa4bef16b12b1fc16111891088bfaf0cd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 16:15:41 +0200 Subject: [PATCH 015/139] fixins --- node/core/approval-voting/src/lib.rs | 35 +++------------ node/core/av-store/src/lib.rs | 42 +++-------------- node/core/candidate-validation/src/lib.rs | 14 +----- node/core/chain-api/src/lib.rs | 14 +----- node/core/chain-selection/src/lib.rs | 55 +++++++++-------------- 5 files changed, 35 insertions(+), 125 deletions(-) diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 9ade68e48696..7c509a7482e5 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -357,12 +357,7 @@ impl ApprovalVotingSubsystem { impl overseer::Subsystem for ApprovalVotingSubsystem where - Context: overseer::SubsystemContext< - Message = ApprovalVotingMessage, - OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ApprovalVotingContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); @@ -713,12 +708,7 @@ async fn run( mut backend: B, ) -> SubsystemResult<()> where - Context: overseer::SubsystemContext< - Message = ApprovalVotingMessage, - OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ApprovalVotingContextTrait, B: Backend, { let mut state = State { @@ -857,12 +847,7 @@ where // // returns `true` if any of the actions was a `Conclude` command. async fn handle_actions( - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalVotingMessage, - OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalVotingContextTrait, state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, @@ -1099,12 +1084,7 @@ fn distribution_messages_for_activation( // Handle an incoming signal from the overseer. Returns true if execution should conclude. async fn handle_from_overseer( - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalVotingMessage, - OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalVotingContextTrait, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, @@ -2164,12 +2144,7 @@ fn process_wakeup( // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. async fn launch_approval( - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalVotingMessage, - OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalVotingContextTrait, metrics: Metrics, session_index: SessionIndex, candidate: CandidateReceipt, diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 20e055672dcd..50bb5ab9f8fa 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -521,12 +521,7 @@ impl KnownUnfinalizedBlocks { impl overseer::Subsystem for AvailabilityStoreSubsystem where - Context: overseer::SubsystemContext< - Message = AvailabilityStoreMessage, - OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityStoreContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(self, ctx).map(|_| Ok(())).boxed(); @@ -537,12 +532,7 @@ where async fn run(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context) where - Context: overseer::SubsystemContext< - Message = AvailabilityStoreMessage, - OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityStoreContextTrait, { let mut next_pruning = Delay::new(subsystem.pruning_config.pruning_interval).fuse(); @@ -570,12 +560,7 @@ async fn run_iteration( mut next_pruning: &mut future::Fuse, ) -> Result where - Context: overseer::SubsystemContext< - Message = AvailabilityStoreMessage, - OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityStoreContextTrait, { select! { incoming = ctx.recv().fuse() => { @@ -626,12 +611,7 @@ async fn process_block_activated( activated: Hash, ) -> Result<(), Error> where - Context: overseer::SubsystemContext< - Message = AvailabilityStoreMessage, - OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityStoreContextTrait, { let now = subsystem.clock.now()?; @@ -690,12 +670,7 @@ async fn process_new_head( header: Header, ) -> Result<(), Error> where - Context: overseer::SubsystemContext< - Message = AvailabilityStoreMessage, - OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityStoreContextTrait, { let candidate_events = util::request_candidate_events(hash, ctx.sender()).await.await??; @@ -841,12 +816,7 @@ async fn process_block_finalized( finalized_number: BlockNumber, ) -> Result<(), Error> where - Context: overseer::SubsystemContext< - Message = AvailabilityStoreMessage, - OutgoingMessages = overseer::AvailabilityStoreOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityStoreContextTrait, { let now = subsystem.clock.now()?; diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index aebf3c42b2bd..f80bf097a192 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -95,12 +95,7 @@ impl CandidateValidationSubsystem { impl overseer::Subsystem for CandidateValidationSubsystem where - Context: overseer::SubsystemContext< - Message = CandidateValidationMessage, - OutgoingMessages = overseer::CandidateValidationOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CandidateValidationContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run( @@ -124,12 +119,7 @@ async fn run( program_path: PathBuf, ) -> SubsystemResult<()> where - Context: overseer::SubsystemContext< - Message = CandidateValidationMessage, - OutgoingMessages = overseer::CandidateValidationOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CandidateValidationContextTrait, { let (validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 76d9ec697a66..7e4240c93517 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -67,12 +67,7 @@ impl ChainApiSubsystem { impl overseer::Subsystem for ChainApiSubsystem where Client: HeaderBackend + AuxStore + 'static, - Context: overseer::SubsystemContext< - Message = ChainApiMessage, - OutgoingMessages = overseer::ChainApiOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ChainApiContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(ctx, self) @@ -88,12 +83,7 @@ async fn run( ) -> SubsystemResult<()> where Client: HeaderBackend + AuxStore, - Context: overseer::SubsystemContext< - Message = ChainApiMessage, - OutgoingMessages = overseer::ChainApiOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ChainApiContextTrait, { loop { match ctx.recv().await? { diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index febf349515d8..9b02c8e1bf07 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -20,7 +20,7 @@ use polkadot_node_primitives::BlockWeight; use polkadot_node_subsystem::{ errors::ChainApiError, messages::{ChainApiMessage, ChainSelectionMessage}, - overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, + overseer::{self, SubsystemSender}, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, }; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::v2::{BlockNumber, ConsensusLog, Hash, Header}; @@ -330,12 +330,7 @@ impl ChainSelectionSubsystem { impl overseer::Subsystem for ChainSelectionSubsystem where - Context: overseer::SubsystemContext< - Message = ChainSelectionMessage, - OutgoingMessages = overseer::ChainSelectionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ChainSelectionContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = db_backend::v1::DbBackend::new( @@ -358,12 +353,7 @@ async fn run( stagnant_check_interval: StagnantCheckInterval, clock: Box, ) where - Context: overseer::SubsystemContext< - Message = ChainSelectionMessage, - OutgoingMessages = overseer::ChainSelectionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ChainSelectionContextTrait, B: Backend, { loop { @@ -394,12 +384,7 @@ async fn run_until_error( clock: &(dyn Clock + Sync), ) -> Result<(), Error> where - Context: overseer::SubsystemContext< - Message = ChainSelectionMessage, - OutgoingMessages = overseer::ChainSelectionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ChainSelectionContextTrait, B: Backend, { let mut stagnant_check_stream = stagnant_check_interval.timeout_stream(); @@ -414,7 +399,7 @@ where FromOverseer::Signal(OverseerSignal::ActiveLeaves(update)) => { for leaf in update.activated { let write_ops = handle_active_leaf( - ctx, + ctx.sender(), &*backend, clock.timestamp_now() + STAGNANT_TIMEOUT, leaf.hash, @@ -431,7 +416,7 @@ where handle_approved_block(backend, hash)? } ChainSelectionMessage::Leaves(tx) => { - let leaves = load_leaves(ctx, &*backend).await?; + let leaves = load_leaves(ctx.sender(), &*backend).await?; let _ = tx.send(leaves); } ChainSelectionMessage::BestLeafContaining(required, tx) => { @@ -458,11 +443,11 @@ where } async fn fetch_finalized( - ctx: &mut impl SubsystemContext, + sender: &mut impl SubsystemSender, ) -> Result, Error> { let (number_tx, number_rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::FinalizedBlockNumber(number_tx)).await; + sender.send_message(ChainApiMessage::FinalizedBlockNumber(number_tx)).await; let number = match number_rx.await? { Ok(number) => number, @@ -474,7 +459,7 @@ async fn fetch_finalized( let (hash_tx, hash_rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::FinalizedBlockHash(number, hash_tx)).await; + sender.send_message(ChainApiMessage::FinalizedBlockHash(number, hash_tx)).await; match hash_rx.await? { Err(err) => { @@ -490,11 +475,11 @@ async fn fetch_finalized( } async fn fetch_header( - ctx: &mut impl SubsystemContext, + sender: &mut impl SubsystemSender, hash: Hash, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockHeader(hash, tx)).await; + sender.send_message(ChainApiMessage::BlockHeader(hash, tx)).await; Ok(rx.await?.unwrap_or_else(|err| { gum::warn!(target: LOG_TARGET, ?hash, ?err, "Missing hash for finalized block number"); @@ -503,11 +488,11 @@ async fn fetch_header( } async fn fetch_block_weight( - ctx: &mut impl SubsystemContext, + sender: &mut impl overseer::SubsystemSender, hash: Hash, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockWeight(hash, tx)).await; + sender.send_message(ChainApiMessage::BlockWeight(hash, tx)).await; let res = rx.await?; @@ -519,7 +504,7 @@ async fn fetch_block_weight( // Handle a new active leaf. async fn handle_active_leaf( - ctx: &mut impl SubsystemContext, + sender: &mut impl overseer::ChainSelectionSenderTrait, backend: &impl Backend, stagnant_at: Timestamp, hash: Hash, @@ -531,10 +516,10 @@ async fn handle_active_leaf( // tree. l.saturating_sub(1) }, - None => fetch_finalized(ctx).await?.map_or(1, |(_, n)| n), + None => fetch_finalized(sender).await?.map_or(1, |(_, n)| n), }; - let header = match fetch_header(ctx, hash).await? { + let header = match fetch_header(sender, hash).await? { None => { gum::warn!(target: LOG_TARGET, ?hash, "Missing header for new head"); return Ok(Vec::new()) @@ -543,7 +528,7 @@ async fn handle_active_leaf( }; let new_blocks = polkadot_node_subsystem_util::determine_new_blocks( - ctx.sender(), + sender, |h| backend.load_block_entry(h).map(|b| b.is_some()), hash, &header, @@ -556,7 +541,7 @@ async fn handle_active_leaf( // determine_new_blocks gives blocks in descending order. // for this, we want ascending order. for (hash, header) in new_blocks.into_iter().rev() { - let weight = match fetch_block_weight(ctx, hash).await? { + let weight = match fetch_block_weight(sender, hash).await? { None => { gum::warn!( target: LOG_TARGET, @@ -666,13 +651,13 @@ fn detect_stagnant(backend: &mut impl Backend, now: Timestamp) -> Result<(), Err // Load the leaves from the backend. If there are no leaves, then return // the finalized block. async fn load_leaves( - ctx: &mut impl SubsystemContext, + sender: &mut impl overseer::SubsystemSender, backend: &impl Backend, ) -> Result, Error> { let leaves: Vec<_> = backend.load_leaves()?.into_hashes_descending().collect(); if leaves.is_empty() { - Ok(fetch_finalized(ctx).await?.map_or(Vec::new(), |(h, _)| vec![h])) + Ok(fetch_finalized(sender).await?.map_or(Vec::new(), |(h, _)| vec![h])) } else { Ok(leaves) } From 18c027a409986a966d5030cef2cdd6194bb66726 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 16:26:51 +0200 Subject: [PATCH 016/139] move to new generated trait bounds, not compiling yet --- node/core/runtime-api/src/lib.rs | 14 +-- node/network/approval-distribution/src/lib.rs | 14 +-- .../availability-distribution/src/lib.rs | 7 +- node/network/availability-recovery/src/lib.rs | 34 ++----- node/network/bitfield-distribution/src/lib.rs | 7 +- node/network/bridge/src/lib.rs | 14 +-- .../src/collator_side/mod.rs | 84 +++-------------- node/network/collator-protocol/src/lib.rs | 7 +- .../src/validator_side/mod.rs | 93 +++---------------- node/network/gossip-support/src/lib.rs | 7 +- .../network/statement-distribution/src/lib.rs | 7 +- .../proc-macro/src/impl_subsystem.rs | 16 ++-- node/overseer/src/lib.rs | 2 +- node/subsystem-test-helpers/src/lib.rs | 8 +- 14 files changed, 60 insertions(+), 254 deletions(-) diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 4c7b0372d35c..1a662a1383f8 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -96,12 +96,7 @@ impl overseer::Subsystem for RuntimeAp where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: overseer::SubsystemContext< - Message = RuntimeApiMessage, - OutgoingMessages = overseer::RuntimeApiOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::RuntimeApiContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { future: run(ctx, self).boxed(), name: "runtime-api-subsystem" } @@ -344,12 +339,7 @@ async fn run( where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: overseer::SubsystemContext< - Message = RuntimeApiMessage, - OutgoingMessages = overseer::RuntimeApiOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::RuntimeApiContextTrait, { loop { select! { diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index eadab6f9c59a..42a9844fb7ed 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -1595,12 +1595,7 @@ async fn adjust_required_routing_and_propagate( /// Modify the reputation of a peer based on its behavior. async fn modify_reputation( - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, peer_id: PeerId, rep: Rep, ) { @@ -1738,12 +1733,7 @@ impl ApprovalDistribution { impl overseer::Subsystem for ApprovalDistribution where - Context: overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ApprovalDistributionContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index 842a6e2e2f60..6dff9bd82b2d 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -70,12 +70,7 @@ pub struct IncomingRequestReceivers { impl overseer::Subsystem for AvailabilityDistributionSubsystem where - Context: overseer::SubsystemContext< - Message = AvailabilityDistributionMessage, - OutgoingMessages = overseer::AvailabilityDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityDistributionContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index d8bfd76cca60..45b3271488df 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -177,7 +177,7 @@ impl RequestFromBackers { async fn run( &mut self, params: &RecoveryParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result { gum::trace!( target: LOG_TARGET, @@ -301,7 +301,7 @@ impl RequestChunksFromValidators { async fn launch_parallel_requests( &mut self, params: &RecoveryParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) { let num_requests = self.get_desired_request_count(params.threshold); let candidate_hash = ¶ms.candidate_hash; @@ -486,7 +486,7 @@ impl RequestChunksFromValidators { async fn run( &mut self, params: &RecoveryParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result { let metrics = ¶ms.metrics; @@ -801,12 +801,7 @@ impl Default for State { impl Subsystem for AvailabilityRecoverySubsystem where - Context: overseer::SubsystemContext< - Message = AvailabilityRecoveryMessage, - OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityRecoveryContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -846,12 +841,7 @@ async fn launch_recovery_task( metrics: &Metrics, ) -> error::Result<()> where - Context: overseer::SubsystemContext< - Message = AvailabilityRecoveryMessage, - OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityRecoveryContextTrait, { let candidate_hash = receipt.hash(); @@ -903,12 +893,7 @@ async fn handle_recover( metrics: &Metrics, ) -> error::Result<()> where - Context: overseer::SubsystemContext< - Message = AvailabilityRecoveryMessage, - OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityRecoveryContextTrait, { let candidate_hash = receipt.hash(); @@ -970,12 +955,7 @@ async fn query_full_data( candidate_hash: CandidateHash, ) -> error::Result> where - Context: overseer::SubsystemContext< - Message = AvailabilityRecoveryMessage, - OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityRecoveryContextTrait, { let (tx, rx) = oneshot::channel(); ctx.send_message(AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx)) diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 87da2926774d..aacb0e61b22e 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -774,12 +774,7 @@ async fn send_tracked_gossip_message( impl overseer::Subsystem for BitfieldDistribution where - Context: overseer::SubsystemContext< - Message = BitfieldDistributionMessage, - OutgoingMessages = overseer::BitfieldDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::BitfieldDistributionContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 1cb57025e76a..d0961f5dda92 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -395,12 +395,7 @@ async fn handle_subsystem_messages( metrics: Metrics, ) -> Result<(), UnexpectedAbort> where - Context: overseer::SubsystemContext< - Message = NetworkBridgeMessage, - OutgoingMessages = overseer::NetworkBridgeOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::NetworkBridgeContextTrait, N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, { @@ -1047,12 +1042,7 @@ async fn run_network( where N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, - Context: overseer::SubsystemContext< - Message = NetworkBridgeMessage, - OutgoingMessages = overseer::NetworkBridgeOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::NetworkBridgeContextTrait, { let shared = Shared::default(); diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 7734fa64e6fb..7c26e106b0d4 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -370,12 +370,7 @@ async fn distribute_collation( result_sender: Option>, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let relay_parent = receipt.descriptor.relay_parent; @@ -471,12 +466,7 @@ async fn determine_core( relay_parent: Hash, ) -> Result> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let cores = get_availability_cores(sender, relay_parent).await?; @@ -509,12 +499,7 @@ async fn determine_our_validators( relay_parent: Hash, ) -> Result where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let session_index = runtime.get_session_index_for_child(ctx.sender(), relay_parent).await?; let info = &runtime @@ -544,12 +529,7 @@ where /// Issue a `Declare` collation message to the given `peer`. async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let declare_signature_payload = protocol_v1::declare_signature_payload(&state.local_peer_id); @@ -572,12 +552,7 @@ where /// revoke the previous connection request. async fn connect_to_validators(ctx: &mut Context, validator_ids: Vec) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { // ignore address resolution failure // will reissue a new request on new collation @@ -600,12 +575,7 @@ async fn advertise_collation( relay_parent: Hash, peer: PeerId, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let should_advertise = state .our_validators_groups @@ -666,12 +636,7 @@ async fn process_msg( msg: CollatorProtocolMessage, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { use CollatorProtocolMessage::*; @@ -784,12 +749,7 @@ async fn handle_incoming_peer_message( msg: protocol_v1::CollatorProtocolMessage, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { use protocol_v1::CollatorProtocolMessage::*; @@ -869,12 +829,7 @@ async fn handle_incoming_request( req: IncomingRequest, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let _span = state .span_per_relay_parent @@ -949,12 +904,7 @@ async fn handle_peer_view_change( peer_id: PeerId, view: View, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let current = state.peer_views.entry(peer_id.clone()).or_default(); @@ -975,12 +925,7 @@ async fn handle_network_msg( bridge_message: NetworkBridgeEvent, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { use NetworkBridgeEvent::*; @@ -1073,12 +1018,7 @@ pub(crate) async fn run( metrics: Metrics, ) -> std::result::Result<(), FatalError> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { use OverseerSignal::*; diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 0256d1588af5..998b6752073f 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -118,12 +118,7 @@ impl CollatorProtocolSubsystem { impl overseer::Subsystem for CollatorProtocolSubsystem where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, ::Sender: SubsystemSender, { diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index e4d6160255c0..7ba182f7793b 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -362,7 +362,7 @@ struct ActiveParas { impl ActiveParas { async fn assign_incoming( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, new_relay_parents: impl IntoIterator, ) { @@ -632,12 +632,7 @@ fn collator_peer_id( async fn disconnect_peer(ctx: &mut Context, peer_id: PeerId) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { ctx.send_message(NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation)) .await @@ -650,12 +645,7 @@ async fn fetch_collation( pc: PendingCollation, id: CollatorId, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let (tx, rx) = oneshot::channel(); @@ -711,12 +701,7 @@ async fn note_good_collation( peer_data: &HashMap, id: CollatorId, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(ctx, peer_id, BENEFIT_NOTIFY_GOOD).await; @@ -730,12 +715,7 @@ async fn notify_collation_seconded( relay_parent: Hash, statement: SignedFullStatement, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let wire_message = protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement.into()); @@ -776,12 +756,7 @@ async fn request_collation( peer_id: PeerId, result: oneshot::Sender<(CandidateReceipt, PoV)>, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { if !state.view.contains(&relay_parent) { gum::debug!( @@ -847,12 +822,7 @@ async fn process_incoming_peer_message( origin: PeerId, msg: protocol_v1::CollatorProtocolMessage, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { use protocol_v1::CollatorProtocolMessage::*; use sp_runtime::traits::AppVerify; @@ -1040,12 +1010,7 @@ async fn handle_our_view_change( view: OurView, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let old_view = std::mem::replace(&mut state.view, view); @@ -1103,12 +1068,7 @@ async fn handle_network_msg( bridge_message: NetworkBridgeEvent, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { use NetworkBridgeEvent::*; @@ -1145,12 +1105,7 @@ async fn process_msg( msg: CollatorProtocolMessage, state: &mut State, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { use CollatorProtocolMessage::*; @@ -1252,12 +1207,7 @@ pub(crate) async fn run( metrics: Metrics, ) -> std::result::Result<(), crate::error::FatalError> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { let mut state = State { metrics, ..Default::default() }; @@ -1343,12 +1293,7 @@ async fn poll_requests( /// Dequeue another collation and fetch. async fn dequeue_next_collation_and_fetch( - ctx: &mut impl overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::CollatorProtocolContextTrait, state: &mut State, relay_parent: Hash, // The collator we tried to fetch from last. @@ -1382,12 +1327,7 @@ async fn handle_collation_fetched_result( state: &mut State, (mut collation_event, res): PendingCollationFetch, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { // If no prior collation for this relay parent has been seconded, then // memorize the `collation_event` for that `relay_parent`, such that we may @@ -1452,12 +1392,7 @@ async fn disconnect_inactive_peers( eviction_policy: &crate::CollatorEvictionPolicy, peers: &HashMap, ) where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { for (peer, peer_data) in peers { if peer_data.is_inactive(&eviction_policy) { diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 83902942a412..2cb086a72e4d 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -588,12 +588,7 @@ fn matrix_neighbors( impl overseer::Subsystem for GossipSupport where - Context: overseer::SubsystemContext< - Message = GossipSupportMessage, - OutgoingMessages = overseer::GossipSupportOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::GossipSupportContextTrait, AD: AuthorityDiscovery + Clone, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 0ee8e825f8cf..d6c12fb9ca97 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -130,12 +130,7 @@ pub struct StatementDistributionSubsystem { impl overseer::Subsystem for StatementDistributionSubsystem where - Context: overseer::SubsystemContext< - Message = StatementDistributionMessage, - OutgoingMessages = overseer::StatementDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::StatementDistributionContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 5e4aa008fdad..cc6e9c3e9ebc 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -167,7 +167,7 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let subsystem_ctx_trait= &Ident::new(&(subsystem_name.clone() + "ContextTrait"), span); let subsystem_sender_trait= &Ident::new(&(subsystem_name.clone() + "SenderTrait"), span); - ts.extend(impl_per_subsystem_trait( + ts.extend(impl_per_subsystem_helper_traits( info, subsystem_ctx_trait, subsystem_sender_name, @@ -359,7 +359,7 @@ pub(crate) fn impl_associate_outgoing_messages( } -pub(crate) fn impl_per_subsystem_trait( +pub(crate) fn impl_per_subsystem_helper_traits( info: &OverseerInfo, subsystem_ctx_trait: &Ident, subsystem_sender_name: &Ident, @@ -375,6 +375,8 @@ pub(crate) fn impl_per_subsystem_trait( let mut ts = TokenStream::new(); + // Create a helper trait bound of all outgoing messages, and the generated wrapper type + // for ease of use within subsystems: let acc_sender_trait_bounds = quote !{ #support_crate ::SubsystemSender< #outgoing_wrapper > #( @@ -394,12 +396,12 @@ pub(crate) fn impl_per_subsystem_trait( }); - + // Create a helper accumulated per subsystem trait bound: let where_clause = quote !{ - #consumes: AssociateOutgoing, - // : <#consumes as AssociateOutgoing>::OutgoingMessages, + #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, + #all_messages_wrapper: From< #outgoing_wrapper >, #all_messages_wrapper: From< #consumes >, - #subsystem_sender_name < #outgoing_wrapper >: #subsystem_sender_trait + #acc_sender_trait_bounds, + Self::Sender: #subsystem_sender_trait + #acc_sender_trait_bounds, }; ts.extend(quote!{ @@ -408,6 +410,7 @@ pub(crate) fn impl_per_subsystem_trait( Message = #consumes, Signal = #signal, OutgoingMessages = #outgoing_wrapper, + // FIXME setting sender breaks testing, which wants to override this Sender = #subsystem_sender_name < #outgoing_wrapper > , Error = #error_ty, > @@ -421,6 +424,7 @@ pub(crate) fn impl_per_subsystem_trait( Message = #consumes, Signal = #signal, OutgoingMessages = #outgoing_wrapper, + // FIXME setting sender breaks testing Sender = #subsystem_sender_name < #outgoing_wrapper >, Error = #error_ty, >, diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 9ff952e41877..9f8e322c1c37 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -496,7 +496,7 @@ pub struct Overseer { #[subsystem(no_dispatch, DisputeDistributionMessage, sends: [])] dispute_distribution: DisputeDistribution, - #[subsystem(no_dispatch, blocking, ChainSelectionMessage, sends: [])] + #[subsystem(no_dispatch, blocking, ChainSelectionMessage, sends: [ChainApiMessage])] chain_selection: ChainSelection, /// External listeners waiting for a hash to be in the active-leave set. diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index d7d1b6d2ff39..5994c24b9b4a 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -152,7 +152,8 @@ pub fn sender_receiver() -> (TestSubsystemSender, mpsc::UnboundedReceiver overseer::SubsystemSender for TestSubsystemSender where - T: Into + Send + 'static, + AllMessages: From, + T: Send + 'static, { async fn send_message(&mut self, msg: T) { self.tx.send(msg.into()).await.expect("test overseer no longer live"); @@ -182,14 +183,15 @@ pub struct TestSubsystemContext { #[async_trait::async_trait] impl overseer::SubsystemContext for TestSubsystemContext where - M: std::fmt::Debug + Send + 'static, + M: overseer::AssociateOutgoing + std::fmt::Debug + Send + 'static, + AllMessages: From<::OutgoingMessages>, AllMessages: From, Spawner: SpawnNamed + Send + 'static, { type Message = M; type Sender = TestSubsystemSender; type Signal = OverseerSignal; - type OutgoingMessages = AllMessages; + type OutgoingMessages = ::OutgoingMessages; type Error = SubsystemError; async fn try_recv(&mut self) -> Result>, ()> { From 1e9e46b4c2544c18e9a537ae11f98406305cf931 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 17:42:52 +0200 Subject: [PATCH 017/139] fixup disputes-coordinator --- .../dispute-coordinator/src/initialized.rs | 43 ++++----------- node/core/dispute-coordinator/src/lib.rs | 23 ++++---- .../src/participation/mod.rs | 30 ++++++----- .../src/participation/queues/mod.rs | 10 ++-- .../src/participation/tests.rs | 6 +-- .../dispute-coordinator/src/scraping/mod.rs | 54 ++++++++++++------- 6 files changed, 79 insertions(+), 87 deletions(-) diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index d67ff56d7bf3..c41eda4b8f80 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -123,12 +123,7 @@ impl Initialized { clock: Box, ) -> FatalResult<()> where - Context: overseer::SubsystemContext< - Message = DisputeCoordinatorMessage, - OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::DisputeCoordinatorContextTrait, B: Backend, { loop { @@ -165,12 +160,7 @@ impl Initialized { clock: &dyn Clock, ) -> Result<()> where - Context: overseer::SubsystemContext< - Message = DisputeCoordinatorMessage, - OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::DisputeCoordinatorContextTrait, B: Backend, { for (priority, request) in participations.drain(..) { @@ -263,12 +253,7 @@ impl Initialized { async fn process_active_leaves_update( &mut self, - ctx: &mut impl overseer::SubsystemContext< - Message = DisputeCoordinatorMessage, - OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, overlay_db: &mut OverlayedBackend<'_, impl Backend>, update: ActiveLeavesUpdate, now: u64, @@ -280,7 +265,7 @@ impl Initialized { if let Some(new_leaf) = update.activated { match self .rolling_session_window - .cache_session_info_for_head(ctx, new_leaf.hash) + .cache_session_info_for_head(ctx.sender(), new_leaf.hash) .await { Err(e) => { @@ -332,12 +317,7 @@ impl Initialized { /// relay chain. async fn process_on_chain_votes( &mut self, - ctx: &mut impl overseer::SubsystemContext< - Message = DisputeCoordinatorMessage, - OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, overlay_db: &mut OverlayedBackend<'_, impl Backend>, votes: ScrapedOnChainVotes, now: u64, @@ -515,7 +495,7 @@ impl Initialized { async fn handle_incoming( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, overlay_db: &mut OverlayedBackend<'_, impl Backend>, message: DisputeCoordinatorMessage, now: Timestamp, @@ -652,7 +632,7 @@ impl Initialized { async fn handle_import_statements( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, overlay_db: &mut OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, candidate_receipt: MaybeCandidateReceipt, @@ -941,7 +921,7 @@ impl Initialized { async fn issue_local_statement( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, overlay_db: &mut OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, candidate_receipt: CandidateReceipt, @@ -1072,12 +1052,7 @@ enum MuxedMessage { impl MuxedMessage { async fn receive( - ctx: &mut impl overseer::SubsystemContext< - Message = DisputeCoordinatorMessage, - OutgoingMessages = overseer::DisputeCoordinatorOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, from_sender: &mut participation::WorkerMessageReceiver, ) -> FatalResult { // We are only fusing here to make `select` happy, in reality we will quit if the stream diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 16b89c06ae41..ae61cfb455a0 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -32,7 +32,8 @@ use sc_keystore::LocalKeystore; use polkadot_node_primitives::{CandidateVotes, DISPUTE_WINDOW}; use polkadot_node_subsystem::{ - messages::DisputeCoordinatorMessage, overseer, ActivatedLeaf, FromOverseer, OverseerSignal, + overseer, + messages::DisputeCoordinatorMessage, ActivatedLeaf, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, }; use polkadot_node_subsystem_util::{ @@ -126,8 +127,7 @@ impl Config { impl overseer::Subsystem for DisputeCoordinatorSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::DisputeCoordinatorContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async { @@ -161,8 +161,7 @@ impl DisputeCoordinatorSubsystem { clock: Box, ) -> FatalResult<()> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::DisputeCoordinatorContextTrait, B: Backend + 'static, { let res = self.initialize(&mut ctx, backend, &*clock).await?; @@ -194,8 +193,7 @@ impl DisputeCoordinatorSubsystem { )>, > where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::DisputeCoordinatorContextTrait, B: Backend + 'static, { loop { @@ -262,8 +260,7 @@ impl DisputeCoordinatorSubsystem { ChainScraper, )> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::DisputeCoordinatorContextTrait, { // Prune obsolete disputes: db::v1::note_current_session(overlay_db, rolling_session_window.latest_session())?; @@ -362,13 +359,12 @@ async fn get_rolling_session_window( ctx: &mut Context, ) -> Result> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::DisputeCoordinatorContextTrait, { if let Some(leaf) = wait_for_first_leaf(ctx).await? { Ok(Some(( leaf.clone(), - RollingSessionWindow::new(ctx, DISPUTE_WINDOW, leaf.hash) + RollingSessionWindow::new(ctx.sender().clone(), DISPUTE_WINDOW, leaf.hash) .await .map_err(JfyiError::RollingSessionWindow)?, ))) @@ -380,8 +376,7 @@ where /// Wait for `ActiveLeavesUpdate`, returns `None` if `Conclude` signal came first. async fn wait_for_first_leaf(ctx: &mut Context) -> Result> where - Context: overseer::SubsystemContext, - Context: SubsystemContext, + Context: overseer::DisputeCoordinatorContextTrait, { loop { match ctx.recv().await? { diff --git a/node/core/dispute-coordinator/src/participation/mod.rs b/node/core/dispute-coordinator/src/participation/mod.rs index 6c47f41cc603..c86f79d5886a 100644 --- a/node/core/dispute-coordinator/src/participation/mod.rs +++ b/node/core/dispute-coordinator/src/participation/mod.rs @@ -27,6 +27,7 @@ use futures_timer::Delay; use polkadot_node_primitives::{ValidationResult, APPROVAL_EXECUTION_TIMEOUT}; use polkadot_node_subsystem::{ + overseer, messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage, CandidateValidationMessage}, ActiveLeavesUpdate, RecoveryError, SubsystemContext, SubsystemSender, }; @@ -144,12 +145,15 @@ impl Participation { /// `on_active_leaves_update`, the participation will be launched right away. /// /// Returns: false, if queues are already full. - pub async fn queue_participation( + pub async fn queue_participation( &mut self, ctx: &mut Context, priority: ParticipationPriority, req: ParticipationRequest, - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeCoordinatorContextTrait, + { // Participation already running - we can ignore that request: if self.running_participations.contains(req.candidate_hash()) { return Ok(()) @@ -174,7 +178,7 @@ impl Participation { /// /// Returns: The received `ParticipationStatement` or a fatal error, in case /// something went wrong when dequeuing more requests (tasks could not be spawned). - pub async fn get_participation_result( + pub async fn get_participation_result( &mut self, ctx: &mut Context, msg: WorkerMessage, @@ -190,7 +194,7 @@ impl Participation { /// /// Make sure we to dequeue participations if that became possible and update most recent /// block. - pub async fn process_active_leaves_update( + pub async fn process_active_leaves_update( &mut self, ctx: &mut Context, update: &ActiveLeavesUpdate, @@ -212,7 +216,7 @@ impl Participation { } /// Dequeue until `MAX_PARALLEL_PARTICIPATIONS` is reached. - async fn dequeue_until_capacity( + async fn dequeue_until_capacity( &mut self, ctx: &mut Context, recent_head: Hash, @@ -228,12 +232,15 @@ impl Participation { } /// Fork a participation task in the background. - fn fork_participation( + fn fork_participation( &mut self, ctx: &mut Context, req: ParticipationRequest, recent_head: Hash, - ) -> FatalResult<()> { + ) -> FatalResult<()> + where + Context: overseer::DisputeCoordinatorContextTrait, + { if self.running_participations.insert(req.candidate_hash().clone()) { let sender = ctx.sender().clone(); ctx.spawn( @@ -248,7 +255,7 @@ impl Participation { async fn participate( mut result_sender: WorkerMessageSender, - mut sender: impl SubsystemSender, + mut sender: impl overseer::DisputeCoordinatorSenderTrait, block_hash: Hash, req: ParticipationRequest, ) { @@ -265,8 +272,7 @@ async fn participate( req.session(), None, recover_available_data_tx, - ) - .into(), + ), ) .await; @@ -332,8 +338,7 @@ async fn participate( n_validators: req.n_validators() as u32, available_data: available_data.clone(), tx: store_available_data_tx, - } - .into(), + }, ) .await; @@ -373,7 +378,6 @@ async fn participate( APPROVAL_EXECUTION_TIMEOUT, validation_tx, ) - .into(), ) .await; diff --git a/node/core/dispute-coordinator/src/participation/queues/mod.rs b/node/core/dispute-coordinator/src/participation/queues/mod.rs index 587f0bf10760..3c7e2c25d55c 100644 --- a/node/core/dispute-coordinator/src/participation/queues/mod.rs +++ b/node/core/dispute-coordinator/src/participation/queues/mod.rs @@ -20,7 +20,7 @@ use std::{ }; use futures::channel::oneshot; -use polkadot_node_subsystem::{messages::ChainApiMessage, SubsystemSender}; +use polkadot_node_subsystem::{messages::ChainApiMessage, overseer, SubsystemSender}; use polkadot_primitives::v2::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex}; use crate::{ @@ -163,7 +163,7 @@ impl Queues { /// Returns error in case a queue was found full already. pub async fn queue( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::DisputeCoordinatorSenderTrait, priority: ParticipationPriority, req: ParticipationRequest, ) -> Result<()> { @@ -305,7 +305,7 @@ impl CandidateComparator { /// `Ok(None)` in case we could not lookup the candidate's relay parent, returns a /// `FatalError` in case the chain API call fails with an unexpected error. pub async fn new( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::DisputeCoordinatorSenderTrait, candidate: &CandidateReceipt, ) -> FatalResult> { let candidate_hash = candidate.hash(); @@ -350,11 +350,11 @@ impl Ord for CandidateComparator { } async fn get_block_number( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::DisputeCoordinatorSenderTrait, relay_parent: Hash, ) -> FatalResult> { let (tx, rx) = oneshot::channel(); - sender.send_message(ChainApiMessage::BlockNumber(relay_parent, tx).into()).await; + sender.send_message(ChainApiMessage::BlockNumber(relay_parent, tx)).await; rx.await .map_err(|_| FatalError::ChainApiSenderDropped)? .map_err(FatalError::ChainApiAncestors) diff --git a/node/core/dispute-coordinator/src/participation/tests.rs b/node/core/dispute-coordinator/src/participation/tests.rs index 7fa635482004..b6c104cc34ec 100644 --- a/node/core/dispute-coordinator/src/participation/tests.rs +++ b/node/core/dispute-coordinator/src/participation/tests.rs @@ -54,7 +54,7 @@ pub fn make_our_subsystem_context( } async fn participate( - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, participation: &mut Participation, ) -> Result<()> { let commitments = CandidateCommitments::default(); @@ -62,7 +62,7 @@ async fn participate( } async fn participate_with_commitments_hash( - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, participation: &mut Participation, commitments_hash: Hash, ) -> Result<()> { @@ -82,7 +82,7 @@ async fn participate_with_commitments_hash( } async fn activate_leaf( - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::DisputeCoordinatorContextTrait, participation: &mut Participation, block_number: BlockNumber, ) -> FatalResult<()> { diff --git a/node/core/dispute-coordinator/src/scraping/mod.rs b/node/core/dispute-coordinator/src/scraping/mod.rs index 060eaecc0414..2920f20bc1fb 100644 --- a/node/core/dispute-coordinator/src/scraping/mod.rs +++ b/node/core/dispute-coordinator/src/scraping/mod.rs @@ -21,7 +21,8 @@ use lru::LruCache; use polkadot_node_primitives::MAX_FINALITY_LAG; use polkadot_node_subsystem::{ - messages::ChainApiMessage, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError, SubsystemSender, + overseer, + messages::{RuntimeApiMessage, ChainApiMessage}, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError, SubsystemSender, }; use polkadot_node_subsystem_util::runtime::{get_candidate_events, get_on_chain_votes}; use polkadot_primitives::v2::{ @@ -81,10 +82,13 @@ impl ChainScraper { /// Create a properly initialized `OrderingProvider`. /// /// Returns: `Self` and any scraped votes. - pub async fn new( + pub async fn new( sender: &mut Sender, initial_head: ActivatedLeaf, - ) -> Result<(Self, Vec)> { + ) -> Result<(Self, Vec)> + where + Sender: overseer::DisputeCoordinatorSenderTrait, + { let mut s = Self { included_candidates: HashSet::new(), candidates_by_block_number: BTreeMap::new(), @@ -106,11 +110,14 @@ impl ChainScraper { /// and updates current heads, so we can query candidates for all non finalized blocks. /// /// Returns: On chain vote for the leaf and any ancestors we might not yet have seen. - pub async fn process_active_leaves_update( + pub async fn process_active_leaves_update( &mut self, sender: &mut Sender, update: &ActiveLeavesUpdate, - ) -> crate::error::Result> { + ) -> crate::error::Result> + where + Sender: overseer::DisputeCoordinatorSenderTrait, + { let activated = match update.activated.as_ref() { Some(activated) => activated, None => return Ok(Vec::new()), @@ -160,12 +167,15 @@ impl ChainScraper { /// Process candidate events of a block. /// /// Keep track of all included candidates. - async fn process_candidate_events( + async fn process_candidate_events( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut Sender, block_number: BlockNumber, block_hash: Hash, - ) -> Result<()> { + ) -> Result<()> + where + Sender: overseer::DisputeCoordinatorSenderTrait, + { // Get included events: let included = get_candidate_events(sender, block_hash) @@ -196,12 +206,15 @@ impl ChainScraper { /// either at the block present in cache or at the last finalized block. /// /// Both `head` and the latest finalized block are **not** included in the result. - async fn get_unfinalized_block_ancestors( + async fn get_unfinalized_block_ancestors( &mut self, sender: &mut Sender, mut head: Hash, mut head_number: BlockNumber, - ) -> Result> { + ) -> Result> + where + Sender: overseer::DisputeCoordinatorSenderTrait, + { let target_ancestor = get_finalized_block_number(sender).await?; let mut ancestors = Vec::new(); @@ -256,16 +269,22 @@ impl ChainScraper { } } -async fn get_finalized_block_number(sender: &mut impl SubsystemSender) -> FatalResult { +async fn get_finalized_block_number(sender: &mut Sender) -> FatalResult +where + Sender: overseer::DisputeCoordinatorSenderTrait, +{ let (number_tx, number_rx) = oneshot::channel(); send_message_fatal(sender, ChainApiMessage::FinalizedBlockNumber(number_tx), number_rx).await } -async fn get_block_ancestors( - sender: &mut impl SubsystemSender, +async fn get_block_ancestors( + sender: &mut Sender, head: Hash, num_ancestors: BlockNumber, -) -> FatalResult> { +) -> FatalResult> +where + Sender: overseer::DisputeCoordinatorSenderTrait, +{ let (tx, rx) = oneshot::channel(); sender .send_message( @@ -273,8 +292,7 @@ async fn get_block_ancestors( hash: head, k: num_ancestors as usize, response_channel: tx, - } - .into(), + }, ) .await; @@ -289,9 +307,9 @@ async fn send_message_fatal( receiver: oneshot::Receiver>, ) -> FatalResult where - Sender: SubsystemSender, + Sender: SubsystemSender, { - sender.send_message(message.into()).await; + sender.send_message(message).await; receiver .await From 1cc4e4eb630257fd5e7938e63bbd2aac4d50f340 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 17:43:30 +0200 Subject: [PATCH 018/139] fixup approval voting --- node/core/approval-voting/src/lib.rs | 8 ++++---- node/core/av-store/src/tests.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 7c509a7482e5..78f386e70f7e 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -604,19 +604,19 @@ impl State { /// Bring `session_window` up to date. pub async fn cache_session_info_for_head( &mut self, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::ApprovalVotingContextTrait, head: Hash, ) -> Result, SessionsUnavailable> { let session_window = self.session_window.take(); match session_window { None => { self.session_window = - Some(RollingSessionWindow::new(ctx, APPROVAL_SESSIONS, head).await?); + Some(RollingSessionWindow::new(ctx.sender(), APPROVAL_SESSIONS, head).await?); Ok(None) }, Some(mut session_window) => { let r = - session_window.cache_session_info_for_head(ctx, head).await.map(Option::Some); + session_window.cache_session_info_for_head(ctx.sender(), head).await.map(Option::Some); self.session_window = Some(session_window); r }, @@ -2364,7 +2364,7 @@ async fn launch_approval( // Issue and import a local approval vote. Should only be invoked after approval checks // have been done. async fn issue_approval( - ctx: &mut impl SubsystemSender, + ctx: &mut impl overseer::ApprovalVotingSenderTrait, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, diff --git a/node/core/av-store/src/tests.rs b/node/core/av-store/src/tests.rs index 8c8a65864f8d..9ef9666d619a 100644 --- a/node/core/av-store/src/tests.rs +++ b/node/core/av-store/src/tests.rs @@ -309,13 +309,13 @@ fn store_chunk_works() { let chunk_msg = AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk: chunk.clone(), tx }; - overseer_send(&mut virtual_overseer, chunk_msg.into()).await; + overseer_send(&mut virtual_overseer, chunk_msg).await; assert_eq!(rx.await.unwrap(), Ok(())); let (tx, rx) = oneshot::channel(); let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx); - overseer_send(&mut virtual_overseer, query_chunk.into()).await; + overseer_send(&mut virtual_overseer, query_chunk).await; assert_eq!(rx.await.unwrap().unwrap(), chunk); virtual_overseer @@ -341,13 +341,13 @@ fn store_chunk_does_nothing_if_no_entry_already() { let chunk_msg = AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk: chunk.clone(), tx }; - overseer_send(&mut virtual_overseer, chunk_msg.into()).await; + overseer_send(&mut virtual_overseer, chunk_msg).await; assert_eq!(rx.await.unwrap(), Err(())); let (tx, rx) = oneshot::channel(); let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx); - overseer_send(&mut virtual_overseer, query_chunk.into()).await; + overseer_send(&mut virtual_overseer, query_chunk).await; assert!(rx.await.unwrap().is_none()); virtual_overseer From 155297284ac73302d56949514eeda37fd934b873 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:12:28 +0200 Subject: [PATCH 019/139] fixup availability distribution --- .../availability-distribution/src/lib.rs | 7 +--- .../src/pov_requester/mod.rs | 7 ++-- .../src/requester/fetch_task/mod.rs | 23 +++++++----- .../src/requester/mod.rs | 35 ++++++++++--------- .../src/requester/session_cache.rs | 9 +++-- node/network/availability-recovery/src/lib.rs | 9 ++--- 6 files changed, 47 insertions(+), 43 deletions(-) diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index 6dff9bd82b2d..964117b043e4 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -96,12 +96,7 @@ impl AvailabilityDistributionSubsystem { /// Start processing work as passed on from the Overseer. async fn run(self, mut ctx: Context) -> std::result::Result<(), FatalError> where - Context: overseer::SubsystemContext< - Message = AvailabilityDistributionMessage, - OutgoingMessages = overseer::AvailabilityDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityDistributionContextTrait, { let Self { mut runtime, recvs, metrics } = self; diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index b1b9af35ef4e..d7269252e2bf 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -24,7 +24,10 @@ use polkadot_node_network_protocol::request_response::{ OutgoingRequest, Recipient, }; use polkadot_node_primitives::PoV; -use polkadot_node_subsystem::{ +use polkadot_node_subsystem_util::runtime::RuntimeInfo; +use polkadot_primitives::v2::{AuthorityDiscoveryId, CandidateHash, Hash, ValidatorIndex}; +use polkadot_subsystem::{ + overseer, jaeger, messages::{IfDisconnected, NetworkBridgeMessage}, SubsystemContext, @@ -50,7 +53,7 @@ pub async fn fetch_pov( metrics: Metrics, ) -> Result<()> where - Context: SubsystemContext, + Context: overseer::AvailabilityDistributionContextTrait, { let info = &runtime.get_session_info(ctx.sender(), parent).await?.session_info; let authority_id = info diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index 9945dd1b4a53..5620d3ed3aca 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -28,7 +28,12 @@ use polkadot_node_network_protocol::request_response::{ v1::{ChunkFetchingRequest, ChunkFetchingResponse}, }; use polkadot_node_primitives::ErasureChunk; -use polkadot_node_subsystem::{ +use polkadot_primitives::v2::{ + AuthorityDiscoveryId, BlakeTwo256, Candi dateHash, GroupIndex, Hash, HashT, OccupiedCore, + SessionIndex, +}; +use polkadot_subsystem::{ + overseer, jaeger, messages::{AllMessages, AvailabilityStoreMessage, IfDisconnected, NetworkBridgeMessage}, SubsystemContext, @@ -84,7 +89,7 @@ enum FetchedState { /// Messages sent from `FetchTask`s to be handled/forwarded. pub enum FromFetchTask { /// Message to other subsystem. - Message(AllMessages), + Message(overseer::AvailabilityDistributionOutgoingMessages), // FIXME TODO /// Concluded with result. /// @@ -177,7 +182,7 @@ impl FetchTask { /// A task handling the fetching of the configured chunk will be spawned. pub async fn start(config: FetchTaskConfig, ctx: &mut Context) -> Result where - Context: SubsystemContext, + Context: overseer::AvailabilityDistributionContextTrait, { let FetchTaskConfig { prepared_running, live_in } = config; @@ -333,9 +338,9 @@ impl RunningTask { let requests = Requests::ChunkFetchingV1(full_request); self.sender - .send(FromFetchTask::Message(AllMessages::NetworkBridge( - NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::ImmediateError), - ))) + .send(FromFetchTask::Message( + NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::ImmediateError).into() + )) .await .map_err(|_| TaskError::ShuttingDown)?; @@ -413,13 +418,13 @@ impl RunningTask { let (tx, rx) = oneshot::channel(); let r = self .sender - .send(FromFetchTask::Message(AllMessages::AvailabilityStore( + .send(FromFetchTask::Message( AvailabilityStoreMessage::StoreChunk { candidate_hash: self.request.candidate_hash, chunk, tx, - }, - ))) + }.into() + )) .await; if let Err(err) = r { gum::error!(target: LOG_TARGET, err= ?err, "Storing erasure chunk failed, system shutting down?"); diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index 7a4043c01a9a..da5913e8e2f6 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -32,8 +32,11 @@ use futures::{ Stream, }; +use polkadot_node_subsystem_util::runtime::{get_occupied_cores, RuntimeInfo}; +use polkadot_primitives::v2::{CandidateHash, Hash, OccupiedCore, SessionIndex}; use polkadot_node_subsystem::{ - messages::{AllMessages, ChainApiMessage}, + overseer, + messages::{RuntimeApiMessage, ChainApiMessage}, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, SubsystemContext, }; use polkadot_node_subsystem_util::runtime::{get_occupied_cores, RuntimeInfo}; @@ -101,7 +104,7 @@ impl Requester { update: ActiveLeavesUpdate, ) -> Result<()> where - Context: SubsystemContext, + Context: overseer::AvailabilityDistributionContextTrait, { gum::trace!(target: LOG_TARGET, ?update, "Update fetching heads"); let ActiveLeavesUpdate { activated, deactivated } = update; @@ -127,12 +130,12 @@ impl Requester { new_head: ActivatedLeaf, ) -> Result<()> where - Context: SubsystemContext, + Context: overseer::AvailabilityDistributionContextTrait, { - let mut sender = ctx.sender(); + let sender = &mut ctx.sender().clone(); let ActivatedLeaf { hash: leaf, .. } = new_head; let (leaf_session_index, ancestors_in_session) = get_block_ancestors_in_same_session( - &mut sender, + sender, runtime, leaf, Self::LEAF_ANCESTRY_LEN_WITHIN_SESSION, @@ -140,7 +143,7 @@ impl Requester { .await?; // Also spawn or bump tasks for candidates in ancestry in the same session. for hash in std::iter::once(leaf).chain(ancestors_in_session) { - let cores = get_occupied_cores(&mut sender, hash).await?; + let cores = get_occupied_cores(sender, hash).await?; gum::trace!( target: LOG_TARGET, occupied_cores = ?cores, @@ -178,14 +181,14 @@ impl Requester { /// passed in leaf might be some later block where the candidate is still pending availability. async fn add_cores( &mut self, - ctx: &mut Context, + context: &mut Context, runtime: &mut RuntimeInfo, leaf: Hash, leaf_session_index: SessionIndex, cores: impl IntoIterator, ) -> Result<()> where - Context: SubsystemContext, + Context: overseer::AvailabilityDistributionContextTrait, { for core in cores { match self.fetches.entry(core.candidate_hash) { @@ -201,7 +204,7 @@ impl Requester { let task_cfg = self .session_cache .with_session_info( - ctx, + context, runtime, // We use leaf here, the relay_parent must be in the same session as the // leaf. This is guaranteed by runtime which ensures that cores are cleared @@ -222,7 +225,7 @@ impl Requester { }); if let Ok(Some(task_cfg)) = task_cfg { - e.insert(FetchTask::start(task_cfg, ctx).await?); + e.insert(FetchTask::start(task_cfg, context).await?); } // Not a validator, nothing to do. }, @@ -233,9 +236,9 @@ impl Requester { } impl Stream for Requester { - type Item = AllMessages; + type Item = overseer::AvailabilityDistributionOutgoingMessages; - fn poll_next(mut self: Pin<&mut Self>, ctx: &mut Context) -> Poll> { + fn poll_next(mut self: Pin<&mut Self>, ctx: &mut Context) -> Poll> { loop { match Pin::new(&mut self.rx).poll_next(ctx) { Poll::Ready(Some(FromFetchTask::Message(m))) => return Poll::Ready(Some(m)), @@ -304,16 +307,16 @@ where } /// Request up to `limit` ancestor hashes of relay parent from the Chain API. -async fn get_block_ancestors( - ctx: &mut overseer::SubsystemSender, +async fn get_block_ancestors( + sender: &mut Sender, relay_parent: Hash, limit: usize, ) -> Result> where - Context: SubsystemContext, + Sender: overseer::SubsystemSender, { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::Ancestors { + sender.send_message(ChainApiMessage::Ancestors { hash: relay_parent, k: limit, response_channel: tx, diff --git a/node/network/availability-distribution/src/requester/session_cache.rs b/node/network/availability-distribution/src/requester/session_cache.rs index e9febc864322..1ff19b9c0c56 100644 --- a/node/network/availability-distribution/src/requester/session_cache.rs +++ b/node/network/availability-distribution/src/requester/session_cache.rs @@ -19,11 +19,14 @@ use std::collections::HashSet; use lru::LruCache; use rand::{seq::SliceRandom, thread_rng}; -use polkadot_node_subsystem::SubsystemContext; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::v2::{ AuthorityDiscoveryId, GroupIndex, Hash, SessionIndex, ValidatorIndex, }; +use polkadot_node_subsystem::{ + SubsystemContext, + overseer, +}; use crate::{ error::{Error, Result}, @@ -103,7 +106,7 @@ impl SessionCache { with_info: F, ) -> Result> where - Context: SubsystemContext, + Context: overseer::AvailabilityDistributionContextTrait, F: FnOnce(&SessionInfo) -> R, { if let Some(o_info) = self.session_info_cache.get(&session_index) { @@ -180,7 +183,7 @@ impl SessionCache { session_index: SessionIndex, ) -> Result> where - Context: SubsystemContext, + Context: overseer::AvailabilityDistributionContextTrait, { let info = runtime .get_session_info_by_index(ctx.sender(), relay_parent, session_index) diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 45b3271488df..717962bca0ec 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -646,7 +646,7 @@ fn reconstructed_data_matches_root( branches.root() == *expected_root } -impl RecoveryTask { +impl RecoveryTask where S: AvailabilityRecoverySenderTrait { async fn run(mut self) -> Result { // First just see if we have the data available locally. { @@ -984,12 +984,7 @@ impl AvailabilityRecoverySubsystem { async fn run(self, mut ctx: Context) -> SubsystemResult<()> where - Context: overseer::SubsystemContext< - Message = AvailabilityRecoveryMessage, - OutgoingMessages = overseer::AvailabilityRecoveryOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::AvailabilityRecoveryContextTrait, { let mut state = State::default(); let Self { fast_path, mut req_receiver, metrics } = self; From 2b1ca3afc9dad93d7cb355c3b5a727281b04d1e3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:18:52 +0200 Subject: [PATCH 020/139] the job based subsystem does not work yet, since the generated trait bound is generated by name, not by a markter type and generic traits over that marker, tbd --- node/core/backing/src/lib.rs | 38 ++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 264e7e27016d..5d83b4602447 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -578,13 +578,13 @@ fn table_attested_to_backed( } async fn store_available_data( - sender: &mut impl SubsystemSender, + ctx: &mut Context, n_validators: u32, candidate_hash: CandidateHash, available_data: AvailableData, ) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); - sender + ctx.sender() .send_message( AvailabilityStoreMessage::StoreAvailableData { candidate_hash, @@ -606,7 +606,7 @@ async fn store_available_data( // This will compute the erasure root internally and compare it to the expected erasure root. // This returns `Err()` iff there is an internal error. Otherwise, it returns either `Ok(Ok(()))` or `Ok(Err(_))`. async fn make_pov_available( - sender: &mut impl SubsystemSender, + ctx: &mut Context, n_validators: usize, pov: Arc, candidate_hash: CandidateHash, @@ -632,21 +632,21 @@ async fn make_pov_available( { let _span = span.as_ref().map(|s| s.child("store-data").with_candidate(candidate_hash)); - store_available_data(sender, n_validators as u32, candidate_hash, available_data).await?; + store_available_data(ctx, n_validators as u32, candidate_hash, available_data).await?; } Ok(Ok(())) } async fn request_pov( - sender: &mut impl SubsystemSender, + ctx: &mut Context, relay_parent: Hash, from_validator: ValidatorIndex, candidate_hash: CandidateHash, pov_hash: Hash, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); - sender + ctx.sender() .send_message( AvailabilityDistributionMessage::FetchPoV { relay_parent, @@ -664,13 +664,13 @@ async fn request_pov( } async fn request_candidate_validation( - sender: &mut impl SubsystemSender, + ctx: &mut impl overseer::CandidateBackingSenderTrait, candidate_receipt: CandidateReceipt, pov: Arc, ) -> Result { let (tx, rx) = oneshot::channel(); - sender + ctx.sender() .send_message( CandidateValidationMessage::ValidateFromChainState( candidate_receipt, @@ -678,7 +678,6 @@ async fn request_candidate_validation( BACKING_EXECUTION_TIMEOUT, tx, ) - .into(), ) .await; @@ -705,7 +704,7 @@ struct BackgroundValidationParams, F> async fn validate_and_make_available( params: BackgroundValidationParams< - impl SubsystemSender, + impl overseer::BackingSenderTrait, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync, >, ) -> Result<(), Error> { @@ -724,7 +723,7 @@ async fn validate_and_make_available( PoVData::Ready(pov) => pov, PoVData::FetchFromValidator { from_validator, candidate_hash, pov_hash } => { let _span = span.as_ref().map(|s| s.child("request-pov")); - match request_pov(&mut sender, relay_parent, from_validator, candidate_hash, pov_hash) + match request_pov(sender, relay_parent, from_validator, candidate_hash, pov_hash) .await { Err(Error::FetchPoV) => { @@ -810,9 +809,6 @@ async fn validate_and_make_available( struct ValidatorIndexOutOfBounds; impl CandidateBackingJob -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, { async fn handle_validated_candidate_command( &mut self, @@ -896,7 +892,7 @@ where &mut self, ctx: &mut Context, params: BackgroundValidationParams< - impl SubsystemSender, + impl overseer::BackingSenderTrait, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync, >, ) -> Result<(), Error> { @@ -1001,7 +997,7 @@ where } /// Check if there have happened any new misbehaviors and issue necessary messages. - fn issue_new_misbehaviors(&mut self, ctx: &mut Context) { + fn issue_new_misbehaviors(&mut self, sender: &mut impl CandidateBackingSenderTrait) { // collect the misbehaviors to avoid double mutable self borrow issues let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect(); for (validator_id, report) in misbehaviors { @@ -1010,7 +1006,7 @@ where // // Misbehaviors are bounded by the number of validators and // the block production protocol. - ctx.send_unbounded_message(ProvisionerMessage::ProvisionableData( + sender.send_unbounded_message(ProvisionerMessage::ProvisionableData( self.parent, ProvisionableData::MisbehaviorReport(self.parent, validator_id, report), )); @@ -1101,7 +1097,7 @@ where None }; - self.issue_new_misbehaviors(ctx); + self.issue_new_misbehaviors(ctx.sender()); // It is important that the child span is dropped before its parent span (`unbacked_span`) drop(import_statement_span); @@ -1123,8 +1119,8 @@ where /// the networking component responsible for feeding statements to the backing subsystem /// is meant to check the signature and provenance of all statements before submission. async fn dispatch_new_statement_to_dispute_coordinator( - &mut self, - ctx: &mut Context, + &self, + ctx: &mut JobSender, candidate_hash: CandidateHash, statement: &SignedFullStatement, ) -> Result<(), ValidatorIndexOutOfBounds> { @@ -1157,7 +1153,7 @@ where if let (Some(candidate_receipt), Some(dispute_statement)) = (maybe_candidate_receipt, maybe_signed_dispute_statement) { - ctx.send_message(DisputeCoordinatorMessage::ImportStatements { + sender.send_message(DisputeCoordinatorMessage::ImportStatements { candidate_hash, candidate_receipt, session: self.session_index, From 0a10c0a2721181ba3975bc477b9dec1f344adff1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:19:51 +0200 Subject: [PATCH 021/139] prep deprecation notice --- node/overseer/overseer-gen/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index b4c4d5efa2db..e1908528b9e9 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -400,6 +400,7 @@ pub trait SubsystemContext: Send + 'static { ) -> Result<(), Self::Error>; /// Send a direct message to some other `Subsystem`, routed based on message type. + // #[deprecated(note = "Use `self.sender().send_message(msg) instead, avoid passing around the full context.")] async fn send_message(&mut self, msg: T) where Self::OutgoingMessages: From + Send, @@ -409,6 +410,7 @@ pub trait SubsystemContext: Send + 'static { } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. + // #[deprecated(note = "Use `self.sender().send_message(msg) instead, avoid passing around the full context.")] async fn send_messages(&mut self, msgs: I) where Self::OutgoingMessages: From + Send, @@ -422,6 +424,7 @@ pub trait SubsystemContext: Send + 'static { } /// Send a message using the unbounded connection. + // #[deprecated(note = "Use `self.sender().send_unbounded_message(msg) instead, avoid passing around the full context.")] fn send_unbounded_message(&mut self, msg: X) where Self::OutgoingMessages: From + Send, From 9bc9f9d2a3cd788be634872e5120b602481e0d4a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:26:46 +0200 Subject: [PATCH 022/139] fixup candidate validation --- node/core/candidate-validation/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index f80bf097a192..2094821365dc 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -419,7 +419,7 @@ pub async fn find_validation_data( descriptor: &CandidateDescriptor, ) -> Result, ValidationFailed> where - Sender: SubsystemSender, + Sender: SubsystemSender, { match find_assumed_validation_data(sender, &descriptor).await { AssumptionCheckOutcome::Matches(validation_data, validation_code) => From 15d209721fcb95df864d9dc337b8e328f337bce2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:27:04 +0200 Subject: [PATCH 023/139] update required outbound messages for the overseer --- node/overseer/src/lib.rs | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 9f8e322c1c37..30057b08453e 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -418,10 +418,15 @@ pub async fn forward_events>(client: Arc

, mut hand message_capacity=2048, )] pub struct Overseer { - #[subsystem(no_dispatch, CandidateValidationMessage, sends: [])] + #[subsystem(no_dispatch, CandidateValidationMessage, sends: [ + RuntimeApiMessage, + ])] candidate_validation: CandidateValidation, - #[subsystem(no_dispatch, PvfCheckerMessage, sends: [CandidateValidationMessage, RuntimeApiMessage])] + #[subsystem(no_dispatch, PvfCheckerMessage, sends: [ + CandidateValidationMessage, + RuntimeApiMessage, + ])] pvf_checker: PvfChecker, #[subsystem(no_dispatch, CandidateBackingMessage, sends: [ @@ -436,10 +441,19 @@ pub struct Overseer { #[subsystem(StatementDistributionMessage, sends: [])] statement_distribution: StatementDistribution, - #[subsystem(no_dispatch, AvailabilityDistributionMessage, sends: [])] + #[subsystem(no_dispatch, AvailabilityDistributionMessage, sends: [ + AvailabilityStoreMessage, + AvailabilityRecoveryMessage, + ChainApiMessage, + RuntimeApiMessage, + NetworkBridgeMessage, + ])] availability_distribution: AvailabilityDistribution, - #[subsystem(no_dispatch, AvailabilityRecoveryMessage, sends: [])] + #[subsystem(no_dispatch, AvailabilityRecoveryMessage, sends: [ + NetworkBridgeMessage, + RuntimeApiMessage, + ])] availability_recovery: AvailabilityRecovery, #[subsystem(blocking, no_dispatch, BitfieldSigningMessage, sends: [])] @@ -448,7 +462,12 @@ pub struct Overseer { #[subsystem(BitfieldDistributionMessage, sends: [])] bitfield_distribution: BitfieldDistribution, - #[subsystem(no_dispatch, ProvisionerMessage, sends: [])] + #[subsystem(no_dispatch, ProvisionerMessage, sends: [ + RuntimeApiMessage, + CandidateBackingMessage, + ChainApiMessage, + DisputeCoordinatorMessage, + ])] provisioner: Provisioner, #[subsystem(no_dispatch, blocking, RuntimeApiMessage, sends: [])] @@ -490,6 +509,10 @@ pub struct Overseer { #[subsystem(no_dispatch, blocking, DisputeCoordinatorMessage, sends: [ RuntimeApiMessage, ChainApiMessage, + DisputeDistributionMessage, + CandidateValidationMessage, + AvailabilityStoreMessage, + AvailabilityRecoveryMessage, ])] dispute_coordinator: DisputeCoordinator, From 0ed6f61a35f110ae1241070a92f5c6f1dbcf1342 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:28:03 +0200 Subject: [PATCH 024/139] fixup subsystem util --- node/subsystem-util/src/lib.rs | 3 +++ node/subsystem-util/src/rolling_session_window.rs | 4 ++-- node/subsystem-util/src/runtime/mod.rs | 2 +- node/subsystem-util/src/tests.rs | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 63c835e6f9c9..66bd44e6b3eb 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -486,6 +486,7 @@ pub trait JobTrait: Unpin + Sized { /// Message type used to send messages to the job. type ToJob: 'static + BoundToRelayParent + Send; + /// The set of outgoing messages to be accumalted into. type OutgoingMessages: 'static + Send; /// Job runtime error. @@ -686,6 +687,8 @@ impl JobSubsystem { OutgoingMessages = ::OutgoingMessages, Signal = OverseerSignal, >, + // FIXME TODO must be bounded on the accumaltive sender trait, + // not on just the outer one ::Sender: SubsystemSender<::OutgoingMessages>, Job: 'static + JobTrait + Send, ::RunArgs: Clone + Sync, diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index f4e782c8389a..32698211e157 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -501,7 +501,7 @@ mod tests { let test_fut = { Box::pin(async move { - let res = RollingSessionWindow::new(&mut ctx, TEST_WINDOW_SIZE, hash).await; + let res = RollingSessionWindow::new(ctx.sender(), TEST_WINDOW_SIZE, hash).await; assert!(res.is_err()); }) }; @@ -561,7 +561,7 @@ mod tests { let test_fut = { Box::pin(async move { let window = - RollingSessionWindow::new(&mut ctx, TEST_WINDOW_SIZE, hash).await.unwrap(); + RollingSessionWindow::new(ctx.sender(), TEST_WINDOW_SIZE, hash).await.unwrap(); assert_eq!(window.earliest_session, session); assert_eq!(window.session_info, vec![dummy_session_info(session)]); diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 14d4909e3d47..47bdb1e97f89 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -322,7 +322,7 @@ pub async fn get_on_chain_votes( relay_parent: Hash, ) -> Result> where - Sender: SubsystemSender, + Sender: SubsystemSender, { recv_runtime(request_on_chain_votes(relay_parent, sender).await).await } diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index d4b9b547008c..0772f367a9cd 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -69,7 +69,7 @@ impl JobTrait for FakeCollatorProtocolJob { /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run( + fn run( _: ActivatedLeaf, run_args: Self::RunArgs, _metrics: Self::Metrics, From 11e77e25d8a1809e9eff76d0dbfe79565ae20251 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:33:01 +0200 Subject: [PATCH 025/139] half fix, dispatch_iter is there to make things efficient, but uses AllMessages which will become an impl detail, not being exposed anymore --- node/network/bridge/src/lib.rs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index d0961f5dda92..a20939612d8e 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -324,12 +324,7 @@ impl Subsystem for NetworkBridge, + Context: overseer::NetworkBridgeContextTrait, { fn start(mut self, ctx: Context) -> SpawnedSubsystem { // The stream of networking events has to be created at initialization, otherwise the @@ -681,7 +676,7 @@ where } async fn handle_network_messages( - mut sender: impl SubsystemSender, + mut sender: impl overseer::NetworkBridgeSenderTrait, mut network_service: impl Network, network_stream: BoxStream<'static, NetworkEvent>, mut authority_discovery_service: AD, @@ -1109,7 +1104,7 @@ fn construct_view( fn update_our_view( net: &mut impl Network, - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::NetworkBridgeContextTrait, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, @@ -1240,21 +1235,21 @@ fn send_collation_message_v1( async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender, + ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { dispatch_validation_events_to_all(std::iter::once(event), ctx).await } async fn dispatch_collation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender, + ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { dispatch_collation_events_to_all(std::iter::once(event), ctx).await } fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender, + ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { for msg in AllMessages::dispatch_iter(event) { ctx.send_unbounded_message(msg); @@ -1263,14 +1258,14 @@ fn dispatch_validation_event_to_all_unbounded( fn dispatch_collation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender, + ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { if let Some(msg) = event.focus().ok().map(CollatorProtocolMessage::NetworkBridgeUpdate) { - ctx.send_unbounded_message(msg.into()); + ctx.send_unbounded_message(msg); } } -async fn dispatch_validation_events_to_all(events: I, ctx: &mut impl SubsystemSender) +async fn dispatch_validation_events_to_all(events: I, ctx: &mut impl overseer::NetworkBridgeSenderTrait) where I: IntoIterator>, I::IntoIter: Send, @@ -1278,7 +1273,7 @@ where ctx.send_messages(events.into_iter().flat_map(AllMessages::dispatch_iter)).await } -async fn dispatch_collation_events_to_all(events: I, ctx: &mut impl SubsystemSender) +async fn dispatch_collation_events_to_all(events: I, ctx: &mut impl overseer::NetworkBridgeSenderTrait) where I: IntoIterator>, I::IntoIter: Send, From fd1d835a19335858f4919b290e7d7ed914932f88 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:48:31 +0200 Subject: [PATCH 026/139] fixup dispute distribution --- node/network/dispute-distribution/src/lib.rs | 42 ++++++------- .../dispute-distribution/src/receiver/mod.rs | 10 +-- .../dispute-distribution/src/sender/mod.rs | 61 +++++++++++++------ .../src/sender/send_task.rs | 13 ++-- 4 files changed, 73 insertions(+), 53 deletions(-) diff --git a/node/network/dispute-distribution/src/lib.rs b/node/network/dispute-distribution/src/lib.rs index 16e8d7767545..24467f40b59b 100644 --- a/node/network/dispute-distribution/src/lib.rs +++ b/node/network/dispute-distribution/src/lib.rs @@ -116,12 +116,7 @@ pub struct DisputeDistributionSubsystem { impl overseer::Subsystem for DisputeDistributionSubsystem where - Context: overseer::SubsystemContext< - Message = DisputeDistributionMessage, - OutgoingMessages = overseer::DisputeDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - > + Context: overseer::DisputeDistributionContextTrait + Sync + Send, AD: AuthorityDiscovery + Clone, @@ -166,13 +161,7 @@ where /// Start processing work as passed on from the Overseer. async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> where - Context: overseer::SubsystemContext< - Message = DisputeDistributionMessage, - OutgoingMessages = overseer::DisputeDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - > + Sync - + Send, + Context: overseer::DisputeDistributionContextTrait + Sync + Send, { let receiver = DisputesReceiver::new( ctx.sender().clone(), @@ -212,11 +201,14 @@ where } /// Handle overseer signals. - async fn handle_signals( + async fn handle_signals( &mut self, ctx: &mut Context, signal: OverseerSignal, - ) -> Result { + ) -> Result + where + Context: overseer::DisputeDistributionContextTrait, + { match signal { OverseerSignal::Conclude => return Ok(SignalResult::Conclude), OverseerSignal::ActiveLeaves(update) => { @@ -228,11 +220,13 @@ where } /// Handle `DisputeDistributionMessage`s. - async fn handle_subsystem_message( + async fn handle_subsystem_message( &mut self, ctx: &mut Context, msg: DisputeDistributionMessage, - ) -> Result<()> { + ) -> Result<()> + where Context: overseer::DisputeDistributionContextTrait, + { match msg { DisputeDistributionMessage::SendDispute(dispute_msg) => self.disputes_sender.start_sender(ctx, &mut self.runtime, dispute_msg).await?, @@ -251,15 +245,13 @@ enum MuxedMessage { } impl MuxedMessage { - async fn receive( - ctx: &mut impl overseer::SubsystemContext< - Message = DisputeDistributionMessage, - OutgoingMessages = overseer::DisputeDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + async fn receive( + ctx: &mut Context, from_sender: &mut mpsc::Receiver, - ) -> Self { + ) -> Self + where + Context: overseer::DisputeDistributionContextTrait, + { // We are only fusing here to make `select` happy, in reality we will quit if the stream // ends. let from_overseer = ctx.recv().fuse(); diff --git a/node/network/dispute-distribution/src/receiver/mod.rs b/node/network/dispute-distribution/src/receiver/mod.rs index ef629e93d6e2..ecc36ac8a2da 100644 --- a/node/network/dispute-distribution/src/receiver/mod.rs +++ b/node/network/dispute-distribution/src/receiver/mod.rs @@ -39,7 +39,9 @@ use polkadot_node_network_protocol::{ PeerId, UnifiedReputationChange as Rep, }; use polkadot_node_primitives::DISPUTE_WINDOW; +use polkadot_node_subsystem_util::{runtime, runtime::RuntimeInfo}; use polkadot_node_subsystem::{ + overseer, messages::{AllMessages, DisputeCoordinatorMessage, ImportStatementsResult}, SubsystemSender, }; @@ -135,7 +137,7 @@ impl MuxedMessage { impl DisputesReceiver where AD: AuthorityDiscovery, - Sender: SubsystemSender, + Sender: overseer::DisputeDistributionSenderTrait, { /// Create a new receiver which can be `run`. pub fn new( @@ -266,15 +268,15 @@ where let (pending_confirmation, confirmation_rx) = oneshot::channel(); let candidate_hash = candidate_receipt.hash(); self.sender - .send_message(AllMessages::DisputeCoordinator( + .send_message( DisputeCoordinatorMessage::ImportStatements { candidate_hash, candidate_receipt, session: valid_vote.0.session_index(), statements: vec![valid_vote, invalid_vote], pending_confirmation: Some(pending_confirmation), - }, - )) + } + ) .await; self.pending_imports.push(peer, confirmation_rx, pending_response); diff --git a/node/network/dispute-distribution/src/sender/mod.rs b/node/network/dispute-distribution/src/sender/mod.rs index b1a21b96ed2e..bf559c2390b8 100644 --- a/node/network/dispute-distribution/src/sender/mod.rs +++ b/node/network/dispute-distribution/src/sender/mod.rs @@ -20,7 +20,10 @@ use futures::channel::{mpsc, oneshot}; use polkadot_node_network_protocol::request_response::v1::DisputeRequest; use polkadot_node_primitives::{CandidateVotes, DisputeMessage, SignedDisputeStatement}; +use polkadot_node_subsystem_util::runtime::RuntimeInfo; +use polkadot_primitives::v2::{CandidateHash, DisputeStatement, Hash, SessionIndex}; use polkadot_node_subsystem::{ + overseer, messages::{AllMessages, DisputeCoordinatorMessage}, ActiveLeavesUpdate, SubsystemContext, }; @@ -79,12 +82,15 @@ impl DisputeSender { } /// Create a `SendTask` for a particular new dispute. - pub async fn start_sender( + pub async fn start_sender( &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, msg: DisputeMessage, - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeDistributionContextTrait, + { let req: DisputeRequest = msg.into(); let candidate_hash = req.0.candidate_receipt.hash(); match self.disputes.entry(candidate_hash) { @@ -114,12 +120,15 @@ impl DisputeSender { /// - Get new authorities to send messages to. /// - Get rid of obsolete tasks and disputes. /// - Get dispute sending started in case we missed one for some reason (e.g. on node startup) - pub async fn update_leaves( + pub async fn update_leaves( &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, update: ActiveLeavesUpdate, - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeDistributionContextTrait, + { let ActiveLeavesUpdate { activated, deactivated } = update; let deactivated: HashSet<_> = deactivated.into_iter().collect(); self.active_heads.retain(|h| !deactivated.contains(h)); @@ -179,12 +188,15 @@ impl DisputeSender { /// Call `start_sender` on all passed in disputes. /// /// Recover necessary votes for building up `DisputeMessage` and start sending for all of them. - async fn start_send_for_dispute( + async fn start_send_for_dispute( &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, dispute: (SessionIndex, CandidateHash), - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeDistributionContextTrait, + { let (session_index, candidate_hash) = dispute; // A relay chain head is required as context for receiving session info information from runtime and // storage. We will iterate `active_sessions` to find a suitable head. We assume that there is at @@ -303,11 +315,14 @@ impl DisputeSender { /// Make active sessions correspond to currently active heads. /// /// Returns: true if sessions changed. - async fn refresh_sessions( + async fn refresh_sessions( &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, - ) -> Result { + ) -> Result + where + Context: overseer::DisputeDistributionContextTrait, + { let new_sessions = get_active_session_indices(ctx, runtime, &self.active_heads).await?; let new_sessions_raw: HashSet<_> = new_sessions.keys().collect(); let old_sessions_raw: HashSet<_> = self.active_sessions.keys().collect(); @@ -321,11 +336,14 @@ impl DisputeSender { /// Retrieve the currently active sessions. /// /// List is all indices of all active sessions together with the head that was used for the query. -async fn get_active_session_indices( +async fn get_active_session_indices( ctx: &mut Context, runtime: &mut RuntimeInfo, active_heads: &Vec, -) -> Result> { +) -> Result> +where + Context: overseer::DisputeDistributionContextTrait, +{ let mut indeces = HashMap::new(); // Iterate all heads we track as active and fetch the child' session indices. for head in active_heads { @@ -336,28 +354,35 @@ async fn get_active_session_indices( } /// Retrieve Set of active disputes from the dispute coordinator. -async fn get_active_disputes( +async fn get_active_disputes( ctx: &mut Context, -) -> JfyiErrorResult> { +) -> JfyiErrorResult> +where + Context: overseer::DisputeDistributionContextTrait, +{ let (tx, rx) = oneshot::channel(); + // Caller scope is in `update_leaves` and this is bounded by fork count. - ctx.send_unbounded_message(AllMessages::DisputeCoordinator( + ctx.send_unbounded_message( DisputeCoordinatorMessage::ActiveDisputes(tx), - )); + ); rx.await.map_err(|_| JfyiError::AskActiveDisputesCanceled) } /// Get all locally available dispute votes for a given dispute. -async fn get_candidate_votes( +async fn get_candidate_votes( ctx: &mut Context, session_index: SessionIndex, candidate_hash: CandidateHash, -) -> JfyiErrorResult> { +) -> JfyiErrorResult> +where + Context: overseer::DisputeDistributionContextTrait, +{ let (tx, rx) = oneshot::channel(); // Caller scope is in `update_leaves` and this is bounded by fork count. - ctx.send_unbounded_message(AllMessages::DisputeCoordinator( + ctx.send_unbounded_message( DisputeCoordinatorMessage::QueryCandidateVotes(vec![(session_index, candidate_hash)], tx), - )); + ); rx.await .map(|v| v.get(0).map(|inner| inner.to_owned().2)) .map_err(|_| JfyiError::AskCandidateVotesCanceled) diff --git a/node/network/dispute-distribution/src/sender/send_task.rs b/node/network/dispute-distribution/src/sender/send_task.rs index 9cf639e2e76b..f1457527a523 100644 --- a/node/network/dispute-distribution/src/sender/send_task.rs +++ b/node/network/dispute-distribution/src/sender/send_task.rs @@ -27,7 +27,8 @@ use polkadot_node_network_protocol::{ IfDisconnected, }; use polkadot_node_subsystem::{ - messages::{AllMessages, NetworkBridgeMessage}, + overseer, + messages::NetworkBridgeMessage, SubsystemContext, }; use polkadot_node_subsystem_util::{metrics, runtime::RuntimeInfo}; @@ -102,7 +103,7 @@ impl TaskResult { impl SendTask { /// Initiates sending a dispute message to peers. - pub async fn new( + pub async fn new( ctx: &mut Context, runtime: &mut RuntimeInfo, active_sessions: &HashMap, @@ -120,7 +121,7 @@ impl SendTask { /// /// This function is called at construction and should also be called whenever a session change /// happens and on a regular basis to ensure we are retrying failed attempts. - pub async fn refresh_sends( + pub async fn refresh_sends( &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, @@ -197,7 +198,7 @@ impl SendTask { /// /// This is all parachain validators of the session the candidate occurred and all authorities /// of all currently active sessions, determined by currently active heads. - async fn get_relevant_validators( + async fn get_relevant_validators( &self, ctx: &mut Context, runtime: &mut RuntimeInfo, @@ -241,7 +242,7 @@ impl SendTask { /// Start sending of the given message to all given authorities. /// /// And spawn tasks for handling the response. -async fn send_requests( +async fn send_requests( ctx: &mut Context, tx: mpsc::Sender, receivers: Vec, @@ -271,7 +272,7 @@ async fn send_requests( } let msg = NetworkBridgeMessage::SendRequests(reqs, IfDisconnected::ImmediateError); - ctx.send_message(AllMessages::NetworkBridge(msg)).await; + ctx.send_message(msg).await; Ok(statuses) } From 0574a781f5c81a403d0bfa1b52b7d3ce7a1c053e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 18:48:47 +0200 Subject: [PATCH 027/139] more outgoing --- node/overseer/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 30057b08453e..872e308b5263 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -516,7 +516,11 @@ pub struct Overseer { ])] dispute_coordinator: DisputeCoordinator, - #[subsystem(no_dispatch, DisputeDistributionMessage, sends: [])] + #[subsystem(no_dispatch, DisputeDistributionMessage, sends: [ + RuntimeApiMessage, + DisputeCoordinatorMessage, + NetworkBridgeMessage, + ])] dispute_distribution: DisputeDistribution, #[subsystem(no_dispatch, blocking, ChainSelectionMessage, sends: [ChainApiMessage])] From 6e615c7a17ec643acb415b507ded1d5660c2a853 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 19:11:42 +0200 Subject: [PATCH 028/139] fixup collator protocol --- .../collator-protocol/src/collator_side/mod.rs | 12 +++++------- node/network/collator-protocol/src/lib.rs | 11 ++--------- .../collator-protocol/src/validator_side/mod.rs | 6 +++++- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 7c26e106b0d4..03bc3526e2ec 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -38,7 +38,7 @@ use polkadot_node_network_protocol::{ use polkadot_node_primitives::{CollationSecondedSignal, PoV, Statement}; use polkadot_node_subsystem::{ jaeger, - messages::{CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage}, + messages::{RuntimeApiMessage, CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage}, overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, }; use polkadot_node_subsystem_util::{ @@ -397,7 +397,7 @@ where // Determine which core the para collated-on is assigned to. // If it is not scheduled then ignore the message. - let (our_core, num_cores) = match determine_core(ctx, id, relay_parent).await? { + let (our_core, num_cores) = match determine_core(ctx.sender(), id, relay_parent).await? { Some(core) => core, None => { gum::warn!( @@ -460,13 +460,11 @@ where /// Get the Id of the Core that is assigned to the para being collated on if any /// and the total number of cores. -async fn determine_core( - sender: &mut SubsystemSender, +async fn determine_core( + sender: &mut impl overseer::SubsystemSender, para_id: ParaId, relay_parent: Hash, ) -> Result> -where - Context: overseer::CollatorProtocolContextTrait, { let cores = get_availability_cores(sender, relay_parent).await?; @@ -508,7 +506,7 @@ where .session_info; gum::debug!(target: LOG_TARGET, ?session_index, "Received session info"); let groups = &info.validator_groups; - let rotation_info = get_group_rotation_info(ctx, relay_parent).await?; + let rotation_info = get_group_rotation_info(ctx.sender(), relay_parent).await?; let current_group_index = rotation_info.group_for_core(core_index, cores); let current_validators = groups diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 998b6752073f..7f3e1c210b6f 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -100,12 +100,7 @@ impl CollatorProtocolSubsystem { async fn run(self, ctx: Context) -> std::result::Result<(), error::FatalError> where - Context: overseer::SubsystemContext< - Message = CollatorProtocolMessage, - OutgoingMessages = overseer::CollatorProtocolOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollatorProtocolContextTrait, { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => @@ -119,8 +114,6 @@ impl CollatorProtocolSubsystem { impl overseer::Subsystem for CollatorProtocolSubsystem where Context: overseer::CollatorProtocolContextTrait, - ::Sender: - SubsystemSender, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -135,7 +128,7 @@ where /// Modify the reputation of a peer based on its behavior. async fn modify_reputation(ctx: &mut Context, peer: PeerId, rep: Rep) where - Context: SubsystemContext, + Context: overseer::CollatorProtocolContextTrait, { gum::trace!( target: LOG_TARGET, diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index 7ba182f7793b..401973b24696 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -47,6 +47,7 @@ use polkadot_node_primitives::{PoV, SignedFullStatement}; use polkadot_node_subsystem::{ jaeger, messages::{ + RuntimeApiMessage, CandidateBackingMessage, CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, NetworkBridgeMessage, }, @@ -689,7 +690,10 @@ async fn report_collator( ctx: &mut Context, peer_data: &HashMap, id: CollatorId, -) { +) +where + Context: overseer::CollatorProtocolContextTrait, +{ if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(ctx, peer_id, COST_REPORT_BAD).await; } From 7c5bf06df96aac3021b37c03ac4c7bb6155da99b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 19:11:57 +0200 Subject: [PATCH 029/139] fixup collator proto --- node/overseer/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 872e308b5263..1f8c144683e2 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -491,7 +491,11 @@ pub struct Overseer { #[subsystem(no_dispatch, CollationGenerationMessage, sends: [])] collation_generation: CollationGeneration, - #[subsystem(no_dispatch, CollatorProtocolMessage, sends: [])] + #[subsystem(no_dispatch, CollatorProtocolMessage, sends: [ + NetworkBridgeMessage, + RuntimeApiMessage, + CandidateBackingMessage, + ])] collator_protocol: CollatorProtocol, #[subsystem(ApprovalDistributionMessage, sends: [])] From 3ada1c0ea92d9f3deae03afac0d381f7f82485bb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 22 Apr 2022 19:12:21 +0200 Subject: [PATCH 030/139] prep provisioner, also job based and hence needs more work --- node/core/provisioner/src/lib.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 6c2a69ebf29d..06c6df09c5aa 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -122,7 +122,7 @@ impl JobTrait for ProvisionerJob { /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run( + fn run( leaf: ActivatedLeaf, _: Self::RunArgs, metrics: Self::Metrics, @@ -159,7 +159,7 @@ impl ProvisionerJob { async fn run_loop( mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::ProvisionerSenderTrait, span: PerLeafSpan, ) -> Result<(), Error> { loop { @@ -198,7 +198,7 @@ impl ProvisionerJob { async fn send_inherent_data( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::ProvisionerSenderTrait, return_senders: Vec>, ) { if let Err(err) = send_inherent_data( @@ -276,7 +276,7 @@ async fn send_inherent_data( bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], return_senders: Vec>, - from_job: &mut impl SubsystemSender, + from_job: &mut impl overseer::ProvisionerSenderTrait, metrics: &Metrics, ) -> Result<(), Error> { let availability_cores = request_availability_cores(leaf.hash, from_job) @@ -395,7 +395,7 @@ async fn select_candidates( bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::ProvisionerSenderTrait, ) -> Result, Error> { let block_number = get_block_number_under_construction(relay_parent, sender).await?; @@ -479,7 +479,6 @@ async fn select_candidates( selected_candidates.clone(), tx, ) - .into(), ) .await; let mut candidates = rx.await.map_err(|err| Error::CanceledBackedCandidates(err))?; @@ -531,10 +530,10 @@ async fn select_candidates( /// in the event of an invalid `relay_parent`, returns `Ok(0)` async fn get_block_number_under_construction( relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::ProvisionerSenderTrait, ) -> Result { let (tx, rx) = oneshot::channel(); - sender.send_message(ChainApiMessage::BlockNumber(relay_parent, tx).into()).await; + sender.send_message(ChainApiMessage::BlockNumber(relay_parent, tx)).await; match rx.await.map_err(|err| Error::CanceledBlockNumber(err))? { Ok(Some(n)) => Ok(n + 1), @@ -592,7 +591,7 @@ enum RequestType { /// Request open disputes identified by `CandidateHash` and the `SessionIndex`. async fn request_disputes( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::ProvisionerSenderTrait, active_or_recent: RequestType, ) -> Vec<(SessionIndex, CandidateHash)> { let (tx, rx) = oneshot::channel(); @@ -601,7 +600,7 @@ async fn request_disputes( RequestType::Active => DisputeCoordinatorMessage::ActiveDisputes(tx), }; // Bounded by block production - `ProvisionerMessage::RequestInherentData`. - sender.send_unbounded_message(msg.into()); + sender.send_unbounded_message(msg); let recent_disputes = match rx.await { Ok(r) => r, @@ -615,13 +614,13 @@ async fn request_disputes( /// Request the relevant dispute statements for a set of disputes identified by `CandidateHash` and the `SessionIndex`. async fn request_votes( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::ProvisionerSenderTrait, disputes_to_query: Vec<(SessionIndex, CandidateHash)>, ) -> Vec<(SessionIndex, CandidateHash, CandidateVotes)> { let (tx, rx) = oneshot::channel(); // Bounded by block production - `ProvisionerMessage::RequestInherentData`. sender.send_unbounded_message( - DisputeCoordinatorMessage::QueryCandidateVotes(disputes_to_query, tx).into(), + DisputeCoordinatorMessage::QueryCandidateVotes(disputes_to_query, tx) ); match rx.await { @@ -666,7 +665,7 @@ fn extend_by_random_subset_without_repetition( const MAX_DISPUTES_FORWARDED_TO_RUNTIME: usize = 1_000; async fn select_disputes( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::ProvisionerSenderTrait, metrics: &metrics::Metrics, _leaf: &ActivatedLeaf, ) -> Result { From 11c2b59627e0cb91f1b5cbf76e2c8f46183a5931 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 25 Apr 2022 13:13:30 +0200 Subject: [PATCH 031/139] chore trait based job sysmte works --- node/core/provisioner/src/lib.rs | 15 +++++++--- node/subsystem-util/src/lib.rs | 47 ++++++++++++++++---------------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 06c6df09c5aa..e55e7b128658 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -110,7 +110,7 @@ pub struct ProvisionerJob { #[derive(Debug, Clone, Copy)] pub struct ProvisionerConfig; -impl JobTrait for ProvisionerJob { +impl JobTrait for ProvisionerJob { type ToJob = ProvisionerMessage; type OutgoingMessages = overseer::ProvisionerOutgoingMessages; type Error = Error; @@ -122,13 +122,14 @@ impl JobTrait for ProvisionerJob { /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run( + fn run( leaf: ActivatedLeaf, _: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, mut sender: JobSender, - ) -> Pin> + Send>> { + ) -> Pin> + Send>> + { let span = leaf.span.clone(); async move { let job = ProvisionerJob::new(leaf, metrics, receiver); @@ -804,4 +805,10 @@ async fn select_disputes( } /// The provisioner subsystem. -pub type ProvisionerSubsystem = JobSubsystem; +pub type ProvisionerSubsystem = JobSubsystem< + ProvisionerJob, + Spawner, + overseer::OverseerSender< + overseer::ProvisionerOutgoingMessages, + > +>; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 66bd44e6b3eb..81d7691c2c65 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -482,7 +482,7 @@ impl fmt::Debug for FromJobCommand { /// /// Jobs are instantiated and killed automatically on appropriate overseer messages. /// Other messages are passed along to and from the job via the overseer to other subsystems. -pub trait JobTrait: Unpin + Sized { +pub trait JobTrait: Unpin + Sized { /// Message type used to send messages to the job. type ToJob: 'static + BoundToRelayParent + Send; @@ -508,15 +508,13 @@ pub trait JobTrait: Unpin + Sized { /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run( + fn run( leaf: ActivatedLeaf, run_args: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, - sender: JobSender, - ) -> Pin> + Send>> - where - S: SubsystemSender<::OutgoingMessages>; + sender: JobSender, + ) -> Pin> + Send>>; } /// Error which can be returned by the jobs manager @@ -564,8 +562,8 @@ where metrics: Job::Metrics, sender: Sender, ) where - Job: JobTrait, - Sender: SubsystemSender<::OutgoingMessages>, + Job: JobTrait, + Sender: SubsystemSender<>::OutgoingMessages>, { let hash = leaf.hash; let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -663,13 +661,13 @@ pub struct JobSubsystemParams { /// - On other incoming messages, if they can be converted into `Job::ToJob` and /// include a hash, then they're forwarded to the appropriate individual job. /// - On outgoing messages from the jobs, it forwards them to the overseer. -pub struct JobSubsystem { +pub struct JobSubsystem, Spawner, Sender> { #[allow(missing_docs)] pub params: JobSubsystemParams, - _marker: std::marker::PhantomData, + _marker: std::marker::PhantomData<(Job, Sender)>, } -impl JobSubsystem { +impl, Spawner, Sender> JobSubsystem { /// Create a new `JobSubsystem`. pub fn new(spawner: Spawner, run_args: Job::RunArgs, metrics: Job::Metrics) -> Self { JobSubsystem { @@ -683,18 +681,17 @@ impl JobSubsystem { where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< - Message = ::ToJob, - OutgoingMessages = ::OutgoingMessages, + Message = >::ToJob, + OutgoingMessages = >::OutgoingMessages, Signal = OverseerSignal, + Sender = Sender, >, - // FIXME TODO must be bounded on the accumaltive sender trait, - // not on just the outer one - ::Sender: SubsystemSender<::OutgoingMessages>, - Job: 'static + JobTrait + Send, - ::RunArgs: Clone + Sync, - ::ToJob: + Sender: SubsystemSender<>::OutgoingMessages>, + Job: 'static + JobTrait + Send, + >::RunArgs: Clone + Sync, + >::ToJob: Sync + From<::Message>, - ::Metrics: Sync, + >::Metrics: Sync, { let JobSubsystem { params: JobSubsystemParams { spawner, run_args, metrics }, .. } = self; @@ -761,17 +758,19 @@ impl JobSubsystem { } } -impl Subsystem for JobSubsystem +impl Subsystem for JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< Message = Job::ToJob, Signal = OverseerSignal, - OutgoingMessages = ::OutgoingMessages, + OutgoingMessages = >::OutgoingMessages, + Sender = Sender, >, - Job: 'static + JobTrait + Send, + Sender: SubsystemSender<>::OutgoingMessages>, + Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - ::ToJob: Sync + From<::Message>, + >::ToJob: Sync + From<::Message>, Job::Metrics: Sync, { fn start(self, ctx: Context) -> SpawnedSubsystem { From a4cfe4eb278a7ffb11252c270fb3a17896681c71 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 25 Apr 2022 13:13:40 +0200 Subject: [PATCH 032/139] Revert "chore trait based job sysmte works" This reverts commit 584b75af76eca9bcba71dfaa7f077afe80fb97c9. --- node/core/provisioner/src/lib.rs | 15 +++------- node/subsystem-util/src/lib.rs | 47 ++++++++++++++++---------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index e55e7b128658..06c6df09c5aa 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -110,7 +110,7 @@ pub struct ProvisionerJob { #[derive(Debug, Clone, Copy)] pub struct ProvisionerConfig; -impl JobTrait for ProvisionerJob { +impl JobTrait for ProvisionerJob { type ToJob = ProvisionerMessage; type OutgoingMessages = overseer::ProvisionerOutgoingMessages; type Error = Error; @@ -122,14 +122,13 @@ impl JobTrait for ProvisionerJob { /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run( + fn run( leaf: ActivatedLeaf, _: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, mut sender: JobSender, - ) -> Pin> + Send>> - { + ) -> Pin> + Send>> { let span = leaf.span.clone(); async move { let job = ProvisionerJob::new(leaf, metrics, receiver); @@ -805,10 +804,4 @@ async fn select_disputes( } /// The provisioner subsystem. -pub type ProvisionerSubsystem = JobSubsystem< - ProvisionerJob, - Spawner, - overseer::OverseerSender< - overseer::ProvisionerOutgoingMessages, - > ->; +pub type ProvisionerSubsystem = JobSubsystem; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 81d7691c2c65..66bd44e6b3eb 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -482,7 +482,7 @@ impl fmt::Debug for FromJobCommand { /// /// Jobs are instantiated and killed automatically on appropriate overseer messages. /// Other messages are passed along to and from the job via the overseer to other subsystems. -pub trait JobTrait: Unpin + Sized { +pub trait JobTrait: Unpin + Sized { /// Message type used to send messages to the job. type ToJob: 'static + BoundToRelayParent + Send; @@ -508,13 +508,15 @@ pub trait JobTrait: Unpin + Sized { /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run( + fn run( leaf: ActivatedLeaf, run_args: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, - sender: JobSender, - ) -> Pin> + Send>>; + sender: JobSender, + ) -> Pin> + Send>> + where + S: SubsystemSender<::OutgoingMessages>; } /// Error which can be returned by the jobs manager @@ -562,8 +564,8 @@ where metrics: Job::Metrics, sender: Sender, ) where - Job: JobTrait, - Sender: SubsystemSender<>::OutgoingMessages>, + Job: JobTrait, + Sender: SubsystemSender<::OutgoingMessages>, { let hash = leaf.hash; let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -661,13 +663,13 @@ pub struct JobSubsystemParams { /// - On other incoming messages, if they can be converted into `Job::ToJob` and /// include a hash, then they're forwarded to the appropriate individual job. /// - On outgoing messages from the jobs, it forwards them to the overseer. -pub struct JobSubsystem, Spawner, Sender> { +pub struct JobSubsystem { #[allow(missing_docs)] pub params: JobSubsystemParams, - _marker: std::marker::PhantomData<(Job, Sender)>, + _marker: std::marker::PhantomData, } -impl, Spawner, Sender> JobSubsystem { +impl JobSubsystem { /// Create a new `JobSubsystem`. pub fn new(spawner: Spawner, run_args: Job::RunArgs, metrics: Job::Metrics) -> Self { JobSubsystem { @@ -681,17 +683,18 @@ impl, Spawner, Sender> JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< - Message = >::ToJob, - OutgoingMessages = >::OutgoingMessages, + Message = ::ToJob, + OutgoingMessages = ::OutgoingMessages, Signal = OverseerSignal, - Sender = Sender, >, - Sender: SubsystemSender<>::OutgoingMessages>, - Job: 'static + JobTrait + Send, - >::RunArgs: Clone + Sync, - >::ToJob: + // FIXME TODO must be bounded on the accumaltive sender trait, + // not on just the outer one + ::Sender: SubsystemSender<::OutgoingMessages>, + Job: 'static + JobTrait + Send, + ::RunArgs: Clone + Sync, + ::ToJob: Sync + From<::Message>, - >::Metrics: Sync, + ::Metrics: Sync, { let JobSubsystem { params: JobSubsystemParams { spawner, run_args, metrics }, .. } = self; @@ -758,19 +761,17 @@ impl, Spawner, Sender> JobSubsystem } } -impl Subsystem for JobSubsystem +impl Subsystem for JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< Message = Job::ToJob, Signal = OverseerSignal, - OutgoingMessages = >::OutgoingMessages, - Sender = Sender, + OutgoingMessages = ::OutgoingMessages, >, - Sender: SubsystemSender<>::OutgoingMessages>, - Job: 'static + JobTrait + Send, + Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - >::ToJob: Sync + From<::Message>, + ::ToJob: Sync + From<::Message>, Job::Metrics: Sync, { fn start(self, ctx: Context) -> SpawnedSubsystem { From 11eeeb0bf6b9d25abec1ea64ca00bb864b0cf108 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 25 Apr 2022 14:04:44 +0200 Subject: [PATCH 033/139] minor foo --- node/core/provisioner/src/lib.rs | 67 ++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 06c6df09c5aa..ed33af11639d 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -95,6 +95,53 @@ impl InherentAfter { } } +/// Errors in the provisioner. +#[derive(Debug, Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + Util(#[from] util::Error), + + #[error("failed to get availability cores")] + CanceledAvailabilityCores(#[source] oneshot::Canceled), + + #[error("failed to get persisted validation data")] + CanceledPersistedValidationData(#[source] oneshot::Canceled), + + #[error("failed to get block number")] + CanceledBlockNumber(#[source] oneshot::Canceled), + + #[error("failed to get backed candidates")] + CanceledBackedCandidates(#[source] oneshot::Canceled), + + #[error("failed to get votes on dispute")] + CanceledCandidateVotes(#[source] oneshot::Canceled), + + #[error(transparent)] + ChainApi(#[from] ChainApiError), + + #[error(transparent)] + Runtime(#[from] RuntimeApiError), + + #[error("failed to send message to ChainAPI")] + ChainApiMessageSend(#[source] mpsc::SendError), + + #[error("failed to send message to CandidateBacking to get backed candidates")] + GetBackedCandidatesSend(#[source] mpsc::SendError), + + #[error("failed to send return message with Inherents")] + InherentDataReturnChannel, + + #[error( + "backed candidate does not correspond to selected candidate; check logic in provisioner" + )] + BackedCandidateOrderingProblem, +} + +/// Provisioner run arguments. +#[derive(Debug, Clone, Copy)] +pub struct ProvisionerConfig; + /// A per-relay-parent job for the provisioning subsystem. pub struct ProvisionerJob { leaf: ActivatedLeaf, @@ -106,10 +153,6 @@ pub struct ProvisionerJob { awaiting_inherent: Vec>, } -/// Provisioner run arguments. -#[derive(Debug, Clone, Copy)] -pub struct ProvisionerConfig; - impl JobTrait for ProvisionerJob { type ToJob = ProvisionerMessage; type OutgoingMessages = overseer::ProvisionerOutgoingMessages; @@ -122,18 +165,18 @@ impl JobTrait for ProvisionerJob { /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run( + fn run( leaf: ActivatedLeaf, _: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, - mut sender: JobSender, + mut sender: JobSender, ) -> Pin> + Send>> { let span = leaf.span.clone(); async move { - let job = ProvisionerJob::new(leaf, metrics, receiver); +\ let job = ProvisionerJob::new(leaf, metrics, receiver); - job.run_loop(sender.subsystem_sender(), PerLeafSpan::new(span, "provisioner")) + job.run_loop::(sender.subsystem_sender(), PerLeafSpan::new(span, "provisioner")) .await } .boxed() @@ -157,9 +200,9 @@ impl ProvisionerJob { } } - async fn run_loop( + async fn run_loop( mut self, - sender: &mut impl overseer::ProvisionerSenderTrait, + sender: &mut Sender, span: PerLeafSpan, ) -> Result<(), Error> { loop { @@ -196,9 +239,9 @@ impl ProvisionerJob { Ok(()) } - async fn send_inherent_data( + async fn send_inherent_data ( &mut self, - sender: &mut impl overseer::ProvisionerSenderTrait, + sender: &mut Sender, return_senders: Vec>, ) { if let Err(err) = send_inherent_data( From 6e6570e7ac3cb01b1f20e3d5aa67c8d6d3db20e9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 25 Apr 2022 14:52:44 +0200 Subject: [PATCH 034/139] rename subsyste util --- node/subsystem-util/src/lib.rs | 70 ++++++++++++++++---------------- node/subsystem-util/src/tests.rs | 4 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 66bd44e6b3eb..f21c5084c411 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -400,14 +400,14 @@ impl Drop for AbortOnDrop { } /// A `JobHandle` manages a particular job for a subsystem. -struct JobHandle { +struct JobHandle { _abort_handle: AbortOnDrop, - to_job: mpsc::Sender, + to_job: mpsc::Sender, } -impl JobHandle { +impl JobHandle { /// Send a message to the job. - async fn send_msg(&mut self, msg: ToJob) -> Result<(), Error> { + async fn send_msg(&mut self, msg: Consumes) -> Result<(), Error> { self.to_job.send(msg).await.map_err(Into::into) } } @@ -484,10 +484,14 @@ impl fmt::Debug for FromJobCommand { /// Other messages are passed along to and from the job via the overseer to other subsystems. pub trait JobTrait: Unpin + Sized { /// Message type used to send messages to the job. - type ToJob: 'static + BoundToRelayParent + Send; + type Consumes: 'static + BoundToRelayParent + Send; /// The set of outgoing messages to be accumalted into. - type OutgoingMessages: 'static + Send; + type Outgoing: 'static + Send; + + /// The sender to send outgoing messages. + // The trait bounds are rather minimal. + type Sender: 'static + Send + Clone; /// Job runtime error. type Error: 'static + std::error::Error + Send; @@ -508,15 +512,13 @@ pub trait JobTrait: Unpin + Sized { /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run( + fn run( leaf: ActivatedLeaf, run_args: Self::RunArgs, metrics: Self::Metrics, - receiver: mpsc::Receiver, - sender: JobSender, - ) -> Pin> + Send>> - where - S: SubsystemSender<::OutgoingMessages>; + receiver: mpsc::Receiver, + sender: JobSender, + ) -> Pin> + Send>>; } /// Error which can be returned by the jobs manager @@ -540,16 +542,16 @@ pub enum JobsError { /// - When dropped, aborts all remaining jobs. /// - implements `Stream`, collecting all messages from subordinate jobs. #[pin_project] -struct Jobs { +struct Jobs { spawner: Spawner, - running: HashMap>, + running: HashMap>, outgoing_msgs: SelectAll>, } -impl Jobs +impl Jobs where Spawner: SpawnNamed, - ToJob: Send + 'static, + Consumes: Send + 'static, { /// Create a new Jobs manager which handles spawning appropriate jobs. pub fn new(spawner: Spawner) -> Self { @@ -557,15 +559,14 @@ where } /// Spawn a new job for this `parent_hash`, with whatever args are appropriate. - fn spawn_job( + fn spawn_job( &mut self, leaf: ActivatedLeaf, run_args: Job::RunArgs, metrics: Job::Metrics, - sender: Sender, + sender: Job::Sender, ) where - Job: JobTrait, - Sender: SubsystemSender<::OutgoingMessages>, + Job: JobTrait, { let hash = leaf.hash; let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -612,7 +613,7 @@ where } /// Send a message to the appropriate job for this `parent_hash`. - async fn send_msg(&mut self, parent_hash: Hash, msg: ToJob) { + async fn send_msg(&mut self, parent_hash: Hash, msg: Consumes) { if let Entry::Occupied(mut job) = self.running.entry(parent_hash) { if job.get_mut().send_msg(msg).await.is_err() { job.remove(); @@ -621,7 +622,7 @@ where } } -impl Stream for Jobs +impl Stream for Jobs where Spawner: SpawnNamed, { @@ -636,7 +637,7 @@ where } } -impl stream::FusedStream for Jobs +impl stream::FusedStream for Jobs where Spawner: SpawnNamed, { @@ -660,7 +661,7 @@ pub struct JobSubsystemParams { /// Conceptually, this is very simple: it just loops forever. /// /// - On incoming overseer messages, it starts or stops jobs as appropriate. -/// - On other incoming messages, if they can be converted into `Job::ToJob` and +/// - On other incoming messages, if they can be converted into `Job::Consumes` and /// include a hash, then they're forwarded to the appropriate individual job. /// - On outgoing messages from the jobs, it forwards them to the overseer. pub struct JobSubsystem { @@ -683,22 +684,20 @@ impl JobSubsystem { where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< - Message = ::ToJob, - OutgoingMessages = ::OutgoingMessages, + Message = ::Consumes, + OutgoingMessages = ::Outgoing, Signal = OverseerSignal, + Sender = ::Sender, >, - // FIXME TODO must be bounded on the accumaltive sender trait, - // not on just the outer one - ::Sender: SubsystemSender<::OutgoingMessages>, Job: 'static + JobTrait + Send, ::RunArgs: Clone + Sync, - ::ToJob: + ::Consumes: Sync + From<::Message>, ::Metrics: Sync, { let JobSubsystem { params: JobSubsystemParams { spawner, run_args, metrics }, .. } = self; - let mut jobs = Jobs::::new(spawner); + let mut jobs = Jobs::::new(spawner); loop { select! { @@ -710,7 +709,7 @@ impl JobSubsystem { }))) => { for activated in activated { let sender = ctx.sender().clone(); - jobs.spawn_job::( + jobs.spawn_job::( activated, run_args.clone(), metrics.clone(), @@ -765,13 +764,14 @@ impl Subsystem for JobSubsystem< where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< - Message = Job::ToJob, + Message = Job::Consumes, Signal = OverseerSignal, - OutgoingMessages = ::OutgoingMessages, + OutgoingMessages = ::Outgoing, + Sender = ::Sender, >, Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - ::ToJob: Sync + From<::Message>, + ::Consumes: Sync + From<::Message>, Job::Metrics: Sync, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 0772f367a9cd..8db6c194d6fd 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -58,8 +58,8 @@ enum Error { } impl JobTrait for FakeCollatorProtocolJob { - type ToJob = CollatorProtocolMessage; - type OutgoingMessages = CollatorProtocolOutgoingMessages; + type Consumes = CollatorProtocolMessage; + type Outgoing = CollatorProtocolOutgoingMessages; type Error = Error; type RunArgs = bool; type Metrics = (); From 0503e6174cf4469256b894213eaff05cc947e10d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 25 Apr 2022 14:55:13 +0200 Subject: [PATCH 035/139] fixup collation --- node/collation-generation/src/lib.rs | 33 +++++++++------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index a11440fd97f0..f1f25b02aedb 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -22,7 +22,7 @@ use futures::{channel::mpsc, future::FutureExt, join, select, sink::SinkExt, str use parity_scale_codec::Encode; use polkadot_node_primitives::{AvailableData, CollationGenerationConfig, PoV}; use polkadot_node_subsystem::{ - messages::{AllMessages, CollationGenerationMessage, CollatorProtocolMessage}, + messages::{CollationGenerationMessage, CollatorProtocolMessage}, overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, SubsystemResult, SubsystemSender, }; @@ -73,12 +73,7 @@ impl CollationGenerationSubsystem { /// Otherwise, most are logged and then discarded. async fn run(mut self, mut ctx: Context) where - Context: overseer::SubsystemContext< - Message = CollationGenerationMessage, - OutgoingMessages = overseer::CollationGenerationOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollationGenerationContextTrait, { // when we activate new leaves, we spawn a bunch of sub-tasks, each of which is // expected to generate precisely one message. We don't want to block the main loop @@ -115,10 +110,7 @@ impl CollationGenerationSubsystem { sender: &mpsc::Sender, ) -> bool where - Context: overseer::SubsystemContext< - Message = CollationGenerationMessage, - OutgoingMessages = overseer::CollationGenerationOutgoingMessages, - >, + Context: overseer::CollationGenerationContextTrait, { match incoming { Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { @@ -170,12 +162,7 @@ impl CollationGenerationSubsystem { impl overseer::Subsystem for CollationGenerationSubsystem where - Context: overseer::SubsystemContext< - Message = CollationGenerationMessage, - OutgoingMessages = overseer::CollationGenerationOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::CollationGenerationContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { @@ -188,12 +175,12 @@ where } } -async fn handle_new_activations( +async fn handle_new_activations( config: Arc, activated: impl IntoIterator, ctx: &mut Context, metrics: Metrics, - sender: &mpsc::Sender, + sender: &mpsc::Sender, ) -> crate::error::Result<()> { // follow the procedure from the guide: // https://w3f.github.io/parachain-implementers-guide/node/collators/collation-generation.html @@ -403,9 +390,9 @@ async fn handle_new_activations( metrics.on_collation_generated(); if let Err(err) = task_sender - .send(AllMessages::CollatorProtocol( - CollatorProtocolMessage::DistributeCollation(ccr, pov, result_sender), - )) + .send( + CollatorProtocolMessage::DistributeCollation(ccr, pov, result_sender).into(), + ) .await { gum::warn!( @@ -427,7 +414,7 @@ async fn obtain_current_validation_code_hash( relay_parent: Hash, para_id: ParaId, assumption: OccupiedCoreAssumption, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::CollationGenerationSenderTrait, ) -> Result, crate::error::Error> { use polkadot_node_subsystem::RuntimeApiError; From 4e160c18dc48eb9c514b6d0793b942317394600e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 25 Apr 2022 14:56:45 +0200 Subject: [PATCH 036/139] fixup provisioner (first job based fixup) --- node/core/provisioner/src/lib.rs | 37 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index ed33af11639d..59513fe1b745 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -143,7 +143,7 @@ pub enum Error { pub struct ProvisionerConfig; /// A per-relay-parent job for the provisioning subsystem. -pub struct ProvisionerJob { +pub struct ProvisionerJob { leaf: ActivatedLeaf, receiver: mpsc::Receiver, backed_candidates: Vec, @@ -151,9 +151,13 @@ pub struct ProvisionerJob { metrics: Metrics, inherent_after: InherentAfter, awaiting_inherent: Vec>, + _phantom: std::marker::PhantomData, } -impl JobTrait for ProvisionerJob { +impl JobTrait for ProvisionerJob +where + Sender: overseer::ProvisionerSenderTrait + std::marker::Unpin, +{ type ToJob = ProvisionerMessage; type OutgoingMessages = overseer::ProvisionerOutgoingMessages; type Error = Error; @@ -165,25 +169,29 @@ impl JobTrait for ProvisionerJob { /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run( + fn run ( leaf: ActivatedLeaf, _: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, mut sender: JobSender, - ) -> Pin> + Send>> { + ) -> Pin> + Send>> + { let span = leaf.span.clone(); async move { -\ let job = ProvisionerJob::new(leaf, metrics, receiver); + let job = ProvisionerJob::new(leaf, metrics, receiver); - job.run_loop::(sender.subsystem_sender(), PerLeafSpan::new(span, "provisioner")) + job.run_loop(sender.subsystem_sender(), PerLeafSpan::new(span, "provisioner")) .await } .boxed() } } -impl ProvisionerJob { +impl ProvisionerJob +where + Sender: overseer::ProvisionerSenderTrait, +{ fn new( leaf: ActivatedLeaf, metrics: Metrics, @@ -197,10 +205,11 @@ impl ProvisionerJob { metrics, inherent_after: InherentAfter::new_from_now(), awaiting_inherent: Vec::new(), + _phantom: std::marker::PhantomData::::default(), } } - async fn run_loop( + async fn run_loop( mut self, sender: &mut Sender, span: PerLeafSpan, @@ -239,7 +248,7 @@ impl ProvisionerJob { Ok(()) } - async fn send_inherent_data ( + async fn send_inherent_data ( &mut self, sender: &mut Sender, return_senders: Vec>, @@ -847,4 +856,12 @@ async fn select_disputes( } /// The provisioner subsystem. -pub type ProvisionerSubsystem = JobSubsystem; +pub type ProvisionerSubsystem = + JobSubsystem< + ProvisionerJob< + overseer::OverseerSender< + overseer::ProvisionerOutgoingMessages + > + >, + Spawner + >; From 2905c2e46f8aa86996860a570a6732c237923b0c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 25 Apr 2022 17:51:07 +0200 Subject: [PATCH 037/139] update overseer ougoing collation gen --- node/overseer/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 1f8c144683e2..170ec56f86fb 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -488,7 +488,10 @@ pub struct Overseer { #[subsystem(no_dispatch, blocking, ChainApiMessage, sends: [])] chain_api: ChainApi, - #[subsystem(no_dispatch, CollationGenerationMessage, sends: [])] + #[subsystem(no_dispatch, CollationGenerationMessage, sends: [ + RuntimeApiMessage, + CollatorProtocolMessage, + ])] collation_generation: CollationGeneration, #[subsystem(no_dispatch, CollatorProtocolMessage, sends: [ From cf7ed17bed154988dd69e0a06e3c99dbd4ccfbc7 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:57:53 +0200 Subject: [PATCH 038/139] chore --- node/core/backing/src/lib.rs | 2 +- node/subsystem-util/src/tests.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 5d83b4602447..36f5df90f6d6 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -1120,7 +1120,7 @@ impl CandidateBackingJob /// is meant to check the signature and provenance of all statements before submission. async fn dispatch_new_statement_to_dispute_coordinator( &self, - ctx: &mut JobSender, + ctx: &mut impl overseer::CandidateBackingContextTrait, candidate_hash: CandidateHash, statement: &SignedFullStatement, ) -> Result<(), ValidatorIndexOutOfBounds> { diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 8db6c194d6fd..c08d51197766 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -57,7 +57,10 @@ enum Error { Sending(#[from] mpsc::SendError), } -impl JobTrait for FakeCollatorProtocolJob { +impl JobTrait for FakeCollatorProtocolJob +where + Sender: overseer::CollatorProtocolSender, +{ type Consumes = CollatorProtocolMessage; type Outgoing = CollatorProtocolOutgoingMessages; type Error = Error; From 10527948b7e838d35b21af5d39f905fa2d46ea1c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:58:09 +0200 Subject: [PATCH 039/139] fixup approval voting --- node/core/approval-voting/src/import.rs | 4 ++-- node/core/approval-voting/src/lib.rs | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index d046d626480c..fc6981c7da68 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -108,7 +108,7 @@ enum ImportedBlockInfoError { /// Computes information about the imported block. Returns an error if the info couldn't be extracted. async fn imported_block_info( - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::ApprovalVotingContextTrait, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -320,7 +320,7 @@ pub struct BlockImportedCandidates { /// /// It is the responsibility of the caller to schedule wakeups for each block. pub(crate) async fn handle_new_head( - ctx: &mut impl overseer::SubsystemContext< + ctx: &mut impl overseer::ApprovalVotingContextTrait< OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, >, state: &mut State, diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 78f386e70f7e..f713039ee06b 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -878,7 +878,7 @@ async fn handle_actions( // Note that chaining these iterators is O(n) as we must consume // the prior iterator. let next_actions: Vec = issue_approval( - &mut sender, + ctx, state, overlayed_db, metrics, @@ -1193,13 +1193,16 @@ async fn handle_from_overseer( Ok(actions) } -async fn handle_approved_ancestor( - ctx: &mut impl overseer::SubsystemContext, +async fn handle_approved_ancestor( + ctx: &mut Context, db: &OverlayedBackend<'_, impl Backend>, target: Hash, lower_bound: BlockNumber, wakeups: &Wakeups, -) -> SubsystemResult> { +) -> SubsystemResult> +where + Context: overseer::ApprovalVotingContextTrait, +{ const MAX_TRACING_WINDOW: usize = 200; const ABNORMAL_DEPTH_THRESHOLD: usize = 5; @@ -2244,7 +2247,6 @@ async fn launch_approval( candidate.clone(), false, ) - .into(), ) .await; metrics_guard.take().on_approval_invalid(); @@ -2285,7 +2287,6 @@ async fn launch_approval( APPROVAL_EXECUTION_TIMEOUT, val_tx, ) - .into(), ) .await; @@ -2311,7 +2312,6 @@ async fn launch_approval( candidate.clone(), false, ) - .into(), ) .await; @@ -2336,7 +2336,6 @@ async fn launch_approval( candidate.clone(), false, ) - .into(), ) .await; @@ -2363,14 +2362,17 @@ async fn launch_approval( // Issue and import a local approval vote. Should only be invoked after approval checks // have been done. -async fn issue_approval( - ctx: &mut impl overseer::ApprovalVotingSenderTrait, +async fn issue_approval( + ctx: &mut Context, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, candidate_hash: CandidateHash, ApprovalVoteRequest { validator_index, block_hash }: ApprovalVoteRequest, -) -> SubsystemResult> { +) -> SubsystemResult> +where + Context: overseer::ApprovalVotingContextTrait, +{ let block_entry = match db.load_block_entry(&block_hash)? { Some(b) => b, None => { @@ -2529,7 +2531,6 @@ async fn issue_approval( validator: validator_index, signature: sig, }) - .into(), ); // dispatch to dispute coordinator. From 5f0b3ccefa92b492ebc1bec591c70bae6148cf3f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:58:33 +0200 Subject: [PATCH 040/139] fixup fuckup fixup --- node/core/backing/src/lib.rs | 4 ++-- node/core/bitfield-signing/src/lib.rs | 20 +++++++++++++------- node/core/provisioner/src/lib.rs | 11 ++++------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 36f5df90f6d6..80b76d251712 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -704,7 +704,7 @@ struct BackgroundValidationParams, F> async fn validate_and_make_available( params: BackgroundValidationParams< - impl overseer::BackingSenderTrait, + impl overseer::CandidateBackingSenderTrait, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync, >, ) -> Result<(), Error> { @@ -892,7 +892,7 @@ impl CandidateBackingJob &mut self, ctx: &mut Context, params: BackgroundValidationParams< - impl overseer::BackingSenderTrait, + Sender, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync, >, ) -> Result<(), Error> { diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index 27284b44e40b..a3b36b6d2b49 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -53,7 +53,7 @@ const JOB_DELAY: Duration = Duration::from_millis(1500); const LOG_TARGET: &str = "parachain::bitfield-signing"; /// Each `BitfieldSigningJob` prepares a signed bitfield for a single relay parent. -pub struct BitfieldSigningJob; +pub struct BitfieldSigningJob(std::marker::PhantomData); /// Errors we may encounter in the course of executing the `BitfieldSigningSubsystem`. #[derive(Debug, thiserror::Error)] @@ -182,9 +182,13 @@ async fn construct_availability_bitfield( Ok(AvailabilityBitfield(core_bits)) } -impl JobTrait for BitfieldSigningJob { +impl JobTrait for BitfieldSigningJob +where + Sender: overseer::BitfieldSigningSenderTrait + Unpin, +{ type ToJob = BitfieldSigningMessage; type OutgoingMessages = overseer::BitfieldSigningOutgoingMessages; + type Sender = Sender; type Error = Error; type RunArgs = SyncCryptoStorePtr; type Metrics = Metrics; @@ -192,15 +196,13 @@ impl JobTrait for BitfieldSigningJob { const NAME: &'static str = "bitfield-signing-job"; /// Run a job for the parent block indicated - fn run( + fn run( leaf: ActivatedLeaf, keystore: Self::RunArgs, metrics: Self::Metrics, _receiver: mpsc::Receiver, - mut sender: JobSender, + mut sender: JobSender, ) -> Pin> + Send>> - where - S: SubsystemSender, { let metrics = metrics.clone(); async move { @@ -290,4 +292,8 @@ impl JobTrait for BitfieldSigningJob { } /// `BitfieldSigningSubsystem` manages a number of bitfield signing jobs. -pub type BitfieldSigningSubsystem = JobSubsystem; +pub type BitfieldSigningSubsystem = + JobSubsystem< + BitfieldSigningJob, + Spawner, + >; diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 59513fe1b745..4e7cabc84ef5 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -160,6 +160,7 @@ where { type ToJob = ProvisionerMessage; type OutgoingMessages = overseer::ProvisionerOutgoingMessages; + type Sender = Sender; type Error = Error; type RunArgs = ProvisionerConfig; type Metrics = Metrics; @@ -856,12 +857,8 @@ async fn select_disputes( } /// The provisioner subsystem. -pub type ProvisionerSubsystem = +pub type ProvisionerSubsystem = JobSubsystem< - ProvisionerJob< - overseer::OverseerSender< - overseer::ProvisionerOutgoingMessages - > - >, - Spawner + ProvisionerJob, + Spawner, >; From 2fa17c758e3cf18916cb86196c9d0251c2aa7cc2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:58:49 +0200 Subject: [PATCH 041/139] avail dist fixup --- node/network/approval-distribution/src/lib.rs | 84 ++++--------------- 1 file changed, 16 insertions(+), 68 deletions(-) diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 42a9844fb7ed..560fbdd5e70d 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -324,12 +324,7 @@ enum PendingMessage { impl State { async fn handle_network_msg( &mut self, - ctx: &mut overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), @@ -383,12 +378,7 @@ impl State { async fn handle_new_blocks( &mut self, - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), @@ -434,6 +424,7 @@ impl State { let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( ctx, + &self.peer_views, metrics, &mut self.blocks, &self.topologies, @@ -506,8 +497,7 @@ impl State { async fn handle_new_session_topology( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::ApprovalDistributionContextTrait, session: SessionIndex, topology: SessionGridTopology, ) { @@ -530,12 +520,7 @@ impl State { async fn process_incoming_peer_message( &mut self, - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, metrics: &Metrics, peer_id: PeerId, msg: protocol_v1::ApprovalDistributionMessage, @@ -626,12 +611,7 @@ impl State { // and has an entry in the `PeerData` struct. async fn handle_peer_view_change( &mut self, - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, metrics: &Metrics, peer_id: PeerId, view: View, @@ -663,6 +643,7 @@ impl State { Self::unify_with_peer( ctx, + &self.peer_views, metrics, &mut self.blocks, &self.topologies, @@ -676,8 +657,7 @@ impl State { async fn handle_block_finalized( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::ApprovalDistributionContextTrait, metrics: &Metrics, finalized_number: BlockNumber, ) { @@ -705,12 +685,7 @@ impl State { async fn import_and_circulate_assignment( &mut self, - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCert, @@ -968,12 +943,7 @@ impl State { async fn import_and_circulate_approval( &mut self, - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, @@ -1233,12 +1203,7 @@ impl State { } async fn unify_with_peer( - ctx: &mut impl overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::ApprovalDistributionContextTrait, gossip_peers: &HashSet, metrics: &Metrics, entries: &mut HashMap, @@ -1384,8 +1349,7 @@ impl State { async fn enable_aggression( &mut self, - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::ApprovalDistributionContextTrait, resend: Resend, metrics: &Metrics, ) { @@ -1487,8 +1451,7 @@ impl State { // Note that the required routing of a message can be modified even if the // topology is unknown yet. async fn adjust_required_routing_and_propagate( - ctx: &mut (impl SubsystemContext - + overseer::SubsystemContext), + ctx: &mut impl overseer::ApprovalDistributionContextTrait, blocks: &mut HashMap, topologies: &SessionGridTopologies, block_filter: impl Fn(&mut BlockEntry) -> bool, @@ -1617,12 +1580,7 @@ impl ApprovalDistribution { async fn run(self, ctx: Context) where - Context: overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ApprovalDistributionContextTrait, { let mut state = State::default(); @@ -1640,12 +1598,7 @@ impl ApprovalDistribution { rng: &mut (impl CryptoRng + Rng) ) where - Context: overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ApprovalDistributionContextTrait, { loop { let message = match ctx.recv().await { @@ -1682,12 +1635,7 @@ impl ApprovalDistribution { metrics: &Metrics, rng: &mut (impl CryptoRng + Rng), ) where - Context: overseer::SubsystemContext< - Message = ApprovalDistributionMessage, - OutgoingMessages = overseer::ApprovalDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::ApprovalDistributionContextTrait, { match msg { ApprovalDistributionMessage::NetworkBridgeUpdate(event) => { From ce4aacdc5745fcfc9c3cc07b37247f0d2370a56e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:59:21 +0200 Subject: [PATCH 042/139] avail rec + bitfield dist fixup --- node/network/availability-recovery/src/lib.rs | 35 +++++++++++-------- node/network/bitfield-distribution/src/lib.rs | 33 +++++++++-------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 717962bca0ec..8e6351780256 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -50,7 +50,7 @@ use polkadot_node_primitives::{AvailableData, ErasureChunk}; use polkadot_node_subsystem::{ errors::RecoveryError, jaeger, - messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeMessage}, + messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeMessage, RuntimeApiMessage}, overseer::{self, Subsystem}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, SubsystemResult, SubsystemSender, @@ -156,8 +156,8 @@ enum Source { /// A stateful reconstruction of availability data in reference to /// a candidate hash. -struct RecoveryTask { - sender: S, +struct RecoveryTask { + sender: Sender, /// The parameters of the recovery process. params: RecoveryParams, @@ -177,7 +177,7 @@ impl RequestFromBackers { async fn run( &mut self, params: &RecoveryParams, - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::AvailabilityRecoverySenderTrait, ) -> Result { gum::trace!( target: LOG_TARGET, @@ -204,7 +204,6 @@ impl RequestFromBackers { vec![Requests::AvailableDataFetchingV1(req)], IfDisconnected::ImmediateError, ) - .into(), ) .await; @@ -298,11 +297,14 @@ impl RequestChunksFromValidators { ) } - async fn launch_parallel_requests( + async fn launch_parallel_requests( &mut self, params: &RecoveryParams, - sender: &mut impl SubsystemSender, - ) { + sender: &mut Sender, + ) + where + Sender: overseer::AvailabilityRecoverySenderTrait, + { let num_requests = self.get_desired_request_count(params.threshold); let candidate_hash = ¶ms.candidate_hash; let already_requesting_count = self.requesting_chunks.len(); @@ -359,7 +361,7 @@ impl RequestChunksFromValidators { sender .send_message( - NetworkBridgeMessage::SendRequests(requests, IfDisconnected::ImmediateError).into(), + NetworkBridgeMessage::SendRequests(requests, IfDisconnected::ImmediateError) ) .await; } @@ -483,11 +485,14 @@ impl RequestChunksFromValidators { } } - async fn run( + async fn run( &mut self, params: &RecoveryParams, - sender: &mut impl SubsystemSender, - ) -> Result { + sender: &mut Sender, + ) -> Result + where + Sender: overseer::AvailabilityRecoverySenderTrait, + { let metrics = ¶ms.metrics; // First query the store for any chunks we've got. @@ -495,7 +500,7 @@ impl RequestChunksFromValidators { let (tx, rx) = oneshot::channel(); sender .send_message( - AvailabilityStoreMessage::QueryAllChunks(params.candidate_hash, tx).into(), + AvailabilityStoreMessage::QueryAllChunks(params.candidate_hash, tx) ) .await; @@ -646,7 +651,7 @@ fn reconstructed_data_matches_root( branches.root() == *expected_root } -impl RecoveryTask where S: AvailabilityRecoverySenderTrait { +impl RecoveryTask where Sender: overseer::AvailabilityRecoverySenderTrait { async fn run(mut self) -> Result { // First just see if we have the data available locally. { @@ -654,7 +659,7 @@ impl RecoveryTask where S: AvailabilityRecoverySenderTrait { self.sender .send_message( AvailabilityStoreMessage::QueryAvailableData(self.params.candidate_hash, tx) - .into(), + ) .await; diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index aacb0e61b22e..67e93f8e8fda 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -213,12 +213,7 @@ impl BitfieldDistribution { /// Start processing work as passed on from the Overseer. async fn run(self, ctx: Context) where - Context: overseer::SubsystemContext< - Message = BitfieldDistributionMessage, - OutgoingMessages = overseer::BitfieldDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::BitfieldDistributionContextTrait, { let mut state = ProtocolState::default(); let mut rng = rand::rngs::StdRng::from_entropy(); @@ -320,7 +315,10 @@ impl BitfieldDistribution { } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) { +async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) +where + Context: overseer::BitfieldDistributionContextTrait, +{ gum::trace!(target: LOG_TARGET, ?relay_parent, ?rep, %peer, "reputation change"); ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await @@ -336,7 +334,7 @@ async fn handle_bitfield_distribution( signed_availability: SignedAvailabilityBitfield, rng: &mut (impl CryptoRng + Rng), ) where - Context: SubsystemContext, + Context: overseer::BitfieldDistributionContextTrait, { let _timer = metrics.time_handle_bitfield_distribution(); @@ -400,7 +398,7 @@ async fn relay_message( required_routing: RequiredRouting, rng: &mut (impl CryptoRng + Rng), ) where - Context: SubsystemContext, + Context: overseer::BitfieldDistributionContextTrait, { let relay_parent = message.relay_parent; let span = job_data.span.child("relay-msg"); @@ -487,7 +485,7 @@ async fn process_incoming_peer_message( message: protocol_v1::BitfieldDistributionMessage, rng: &mut (impl CryptoRng + Rng), ) where - Context: SubsystemContext, + Context: overseer::BitfieldDistributionContextTrait, { let protocol_v1::BitfieldDistributionMessage::Bitfield(relay_parent, bitfield) = message; gum::trace!( @@ -605,7 +603,7 @@ async fn handle_network_msg( bridge_message: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), ) where - Context: SubsystemContext, + Context: overseer::BitfieldDistributionContextTrait, { let _timer = metrics.time_handle_network_msg(); @@ -684,8 +682,8 @@ async fn handle_peer_view_change( origin: PeerId, view: View, rng: &mut (impl CryptoRng + Rng), -) where - Context: SubsystemContext, +)where + Context: overseer::BitfieldDistributionContextTrait, { let added = state .peer_views @@ -743,7 +741,10 @@ async fn send_tracked_gossip_message( dest: PeerId, validator: ValidatorId, message: BitfieldGossipMessage, -) { +) +where + Context: overseer::BitfieldDistributionContextTrait, +{ let job_data = if let Some(job_data) = state.per_relay_parent.get_mut(&message.relay_parent) { job_data } else { @@ -788,7 +789,9 @@ async fn query_basics( ctx: &mut Context, relay_parent: Hash, ) -> SubsystemResult, SigningContext)>> -where { +where + Context: overseer::BitfieldDistributionContextTrait, +{ let (validators_tx, validators_rx) = oneshot::channel(); let (session_tx, session_rx) = oneshot::channel(); From e10c90a46e18658c6132f022befd9a54626f6a52 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:59:37 +0200 Subject: [PATCH 043/139] gossip support fixup --- node/network/gossip-support/src/lib.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 2cb086a72e4d..e43301fbe7e7 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -140,12 +140,7 @@ where async fn run(mut self, mut ctx: Context) -> Self where - Context: overseer::SubsystemContext< - Message = GossipSupportMessage, - OutgoingMessages = overseer::GossipSupportOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + Context: overseer::GossipSupportContextTrait, { fn get_connectivity_check_delay() -> Delay { Delay::new(LOW_CONNECTIVITY_WARN_DELAY) @@ -197,7 +192,7 @@ where /// and issue a connection request. async fn handle_active_leaves( &mut self, - sender: &mut impl overseer::SubsystemSender, + sender: &mut impl overseer::GossipSupportSenderTrait, leaves: impl Iterator, ) -> Result<(), util::Error> { for leaf in leaves { @@ -215,7 +210,7 @@ where if let Some((session_index, relay_parent)) = maybe_issue_connection { let session_info = - util::request_session_info(leaf, session_index, ctx.sender()).await.await??; + util::request_session_info(leaf, session_index, sender).await.await??; let session_info = match session_info { Some(s) => s, @@ -272,7 +267,7 @@ where self.update_authority_status_metrics(&session_info).await; update_gossip_topology( - ctx, + sender, our_index, session_info.discovery_keys, relay_parent, @@ -316,11 +311,14 @@ where } } - async fn issue_connection_request( + async fn issue_connection_request( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut Sender, authorities: Vec, - ) { + ) + where + Sender: overseer::GossipSupportSenderTrait, + { let num = authorities.len(); let mut validator_addrs = Vec::with_capacity(authorities.len()); let mut failures = 0; @@ -440,7 +438,7 @@ where // Get the authorities of the past, present, and future. async fn authorities_past_present_future( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::GossipSupportSenderTrait, relay_parent: Hash, ) -> Result, util::Error> { let authorities = util::request_authorities(relay_parent, sender).await.await??; @@ -494,7 +492,7 @@ async fn remove_all_controlled( /// /// [web3]: https://research.web3.foundation/en/latest/polkadot/networking/3-avail-valid.html#topology async fn update_gossip_topology( - sender: &mut impl SubsystemSender, + sender: &mut impl overseer::GossipSupportSenderTrait, our_index: usize, authorities: Vec, relay_parent: Hash, From df9a95ac4f4c654c7567ecd17da8913b3987a238 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:59:50 +0200 Subject: [PATCH 044/139] statement dist fixup --- .../network/statement-distribution/src/lib.rs | 55 +++++++++---------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index d6c12fb9ca97..7d9031275db5 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -38,7 +38,7 @@ use polkadot_node_subsystem_util::{self as util, rand, MIN_GOSSIP_PEERS}; use polkadot_node_subsystem::{ jaeger, messages::{ - AllMessages, CandidateBackingMessage, NetworkBridgeEvent, NetworkBridgeMessage, + CandidateBackingMessage, NetworkBridgeEvent, NetworkBridgeMessage, StatementDistributionMessage, }, overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, PerLeafSpan, SpawnedSubsystem, @@ -631,12 +631,7 @@ enum MuxedMessage { impl MuxedMessage { async fn receive( - ctx: &mut impl overseer::SubsystemContext< - Message = StatementDistributionMessage, - OutgoingMessages = overseer::StatementDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + ctx: &mut impl overseer::StatementDistributionContextTrait, from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, ) -> MuxedMessage { @@ -897,7 +892,7 @@ async fn circulate_statement_and_dependents( gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, relay_parent: Hash, statement: SignedFullStatement, priority_peers: Vec, @@ -1012,7 +1007,7 @@ fn is_statement_large(statement: &SignedFullStatement) -> (bool, Option) async fn circulate_statement<'a>( gossip_peers: &HashSet, peers: &mut HashMap, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, relay_parent: Hash, stored: StoredStatement<'a>, mut priority_peers: Vec, @@ -1084,10 +1079,10 @@ async fn circulate_statement<'a>( statement = ?stored.statement, "Sending statement", ); - ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::SendValidationMessage( + ctx.send_message(NetworkBridgeMessage::SendValidationMessage( peers_to_send.iter().map(|(p, _)| p.clone()).collect(), payload, - ))) + )) .await; } @@ -1101,7 +1096,7 @@ async fn circulate_statement<'a>( async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, relay_parent: Hash, candidate_hash: CandidateHash, active_head: &ActiveHeadData, @@ -1123,10 +1118,10 @@ async fn send_statements_about( statement = ?statement.statement, "Sending statement", ); - ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::SendValidationMessage( + ctx.send_message(NetworkBridgeMessage::SendValidationMessage( vec![peer.clone()], payload, - ))) + )) .await; metrics.on_statement_distributed(); @@ -1137,7 +1132,7 @@ async fn send_statements_about( async fn send_statements( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, @@ -1157,18 +1152,18 @@ async fn send_statements( statement = ?statement.statement, "Sending statement" ); - ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::SendValidationMessage( + ctx.send_message(NetworkBridgeMessage::SendValidationMessage( vec![peer.clone()], payload, - ))) + )) .await; metrics.on_statement_distributed(); } } -async fn report_peer(ctx: &mut impl overseer::SubsystemContext, peer: PeerId, rep: Rep) { - ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::ReportPeer(peer, rep))) +async fn report_peer(ctx: &mut impl overseer::StatementDistributionContextTrait, peer: PeerId, rep: Rep) { + ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)) .await } @@ -1183,7 +1178,7 @@ async fn retrieve_statement_from_message<'a>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, active_head: &'a mut ActiveHeadData, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option { @@ -1275,7 +1270,7 @@ async fn launch_request( meta: StatementMetadata, peer: PeerId, req_sender: mpsc::Sender, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, metrics: &Metrics, ) -> Option { let (task, handle) = @@ -1308,7 +1303,7 @@ async fn handle_incoming_message_and_circulate<'a>( peers: &mut HashMap, active_heads: &'a mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1364,7 +1359,7 @@ async fn handle_incoming_message<'a>( peer_data: &mut PeerData, active_heads: &'a mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1560,7 +1555,7 @@ async fn update_peer_view_and_maybe_send_unlocked( peer: PeerId, gossip_peers: &HashSet, peer_data: &mut PeerData, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, active_heads: &HashMap, new_view: View, metrics: &Metrics, @@ -1601,7 +1596,7 @@ async fn handle_network_update( authorities: &mut HashMap, active_heads: &mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, req_sender: &mpsc::Sender, update: NetworkBridgeEvent, metrics: &Metrics, @@ -1713,7 +1708,7 @@ impl StatementDistributionSubsystem { async fn run( mut self, - mut ctx: impl overseer::SubsystemContext< + mut ctx: impl overseer::StatementDistributionContextTrait< Message = StatementDistributionMessage, OutgoingMessages = overseer::StatementDistributionOutgoingMessages, Signal = OverseerSignal, @@ -1837,7 +1832,7 @@ impl StatementDistributionSubsystem { async fn handle_requester_message( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &mut HashMap, @@ -1901,10 +1896,10 @@ impl StatementDistributionSubsystem { } }, RequesterMessage::SendRequest(req) => { - ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::SendRequests( + ctx.send_message(NetworkBridgeMessage::SendRequests( vec![req], IfDisconnected::ImmediateError, - ))) + )) .await; }, RequesterMessage::GetMorePeers { relay_parent, candidate_hash, tx } => { @@ -1945,7 +1940,7 @@ impl StatementDistributionSubsystem { async fn handle_subsystem_message( &mut self, - ctx: &mut impl overseer::SubsystemContext, + ctx: &mut impl overseer::StatementDistributionContextTrait, runtime: &mut RuntimeInfo, peers: &mut HashMap, gossip_peers: &mut HashSet, From e068fc23d7d83df28b9a846d896ccaf5e79d3893 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 11:59:58 +0200 Subject: [PATCH 045/139] fixup annotations overseer --- node/overseer/src/lib.rs | 43 ++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 170ec56f86fb..1cd6fed3afb9 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -433,12 +433,19 @@ pub struct Overseer { CandidateValidationMessage, CollatorProtocolMessage, AvailabilityDistributionMessage, + AvailabilityStoreMessage, StatementDistributionMessage, ProvisionerMessage, + RuntimeApiMessage, + DisputeCoordinatorMessage, ])] candidate_backing: CandidateBacking, - #[subsystem(StatementDistributionMessage, sends: [])] + #[subsystem(StatementDistributionMessage, sends: [ + NetworkBridgeMessage, + CandidateBackingMessage, + RuntimeApiMessage, + ])] statement_distribution: StatementDistribution, #[subsystem(no_dispatch, AvailabilityDistributionMessage, sends: [ @@ -453,13 +460,22 @@ pub struct Overseer { #[subsystem(no_dispatch, AvailabilityRecoveryMessage, sends: [ NetworkBridgeMessage, RuntimeApiMessage, + AvailabilityStoreMessage, ])] availability_recovery: AvailabilityRecovery, - #[subsystem(blocking, no_dispatch, BitfieldSigningMessage, sends: [])] + #[subsystem(blocking, no_dispatch, BitfieldSigningMessage, sends: [ + AvailabilityStoreMessage, + RuntimeApiMessage, + BitfieldDistributionMessage, + ])] bitfield_signing: BitfieldSigning, - #[subsystem(BitfieldDistributionMessage, sends: [])] + #[subsystem(BitfieldDistributionMessage, sends: [ + RuntimeApiMessage, + NetworkBridgeMessage, + ProvisionerMessage, + ])] bitfield_distribution: BitfieldDistribution, #[subsystem(no_dispatch, ProvisionerMessage, sends: [ @@ -473,7 +489,10 @@ pub struct Overseer { #[subsystem(no_dispatch, blocking, RuntimeApiMessage, sends: [])] runtime_api: RuntimeApi, - #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage, sends: [])] + #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage, sends: [ + ChainApiMessage, + RuntimeApiMessage, + ])] availability_store: AvailabilityStore, #[subsystem(no_dispatch, NetworkBridgeMessage, sends: [ @@ -501,15 +520,27 @@ pub struct Overseer { ])] collator_protocol: CollatorProtocol, - #[subsystem(ApprovalDistributionMessage, sends: [])] + #[subsystem(ApprovalDistributionMessage, sends: [ + NetworkBridgeMessage, + ApprovalVotingMessage, + ])] approval_distribution: ApprovalDistribution, - #[subsystem(no_dispatch, blocking, ApprovalVotingMessage, sends: [])] + #[subsystem(no_dispatch, blocking, ApprovalVotingMessage, sends: [ + RuntimeApiMessage, + ChainApiMessage, + ChainSelectionMessage, + DisputeCoordinatorMessage, + AvailabilityRecoveryMessage, + ApprovalDistributionMessage, + CandidateValidationMessage, + ])] approval_voting: ApprovalVoting, #[subsystem(GossipSupportMessage, sends: [ NetworkBridgeMessage, RuntimeApiMessage, + ChainSelectionMessage, ])] gossip_support: GossipSupport, From 118f9aeafb10c72fa57b9bb1e9f5f3ea81af8f29 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 12:00:14 +0200 Subject: [PATCH 046/139] fixup borked revert of ToJob -> COnsumes -> Tojob --- node/subsystem-util/src/lib.rs | 40 ++++++++++++++++---------------- node/subsystem-util/src/tests.rs | 27 ++++++++++++++------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index f21c5084c411..63ba4d04ccd9 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -484,10 +484,10 @@ impl fmt::Debug for FromJobCommand { /// Other messages are passed along to and from the job via the overseer to other subsystems. pub trait JobTrait: Unpin + Sized { /// Message type used to send messages to the job. - type Consumes: 'static + BoundToRelayParent + Send; + type ToJob: 'static + BoundToRelayParent + Send; /// The set of outgoing messages to be accumalted into. - type Outgoing: 'static + Send; + type OutgoingMessages: 'static + Send; /// The sender to send outgoing messages. // The trait bounds are rather minimal. @@ -516,7 +516,7 @@ pub trait JobTrait: Unpin + Sized { leaf: ActivatedLeaf, run_args: Self::RunArgs, metrics: Self::Metrics, - receiver: mpsc::Receiver, + receiver: mpsc::Receiver, sender: JobSender, ) -> Pin> + Send>>; } @@ -542,16 +542,16 @@ pub enum JobsError { /// - When dropped, aborts all remaining jobs. /// - implements `Stream`, collecting all messages from subordinate jobs. #[pin_project] -struct Jobs { +struct Jobs { spawner: Spawner, - running: HashMap>, + running: HashMap>, outgoing_msgs: SelectAll>, } -impl Jobs +impl Jobs where Spawner: SpawnNamed, - Consumes: Send + 'static, + ToJob: Send + 'static, { /// Create a new Jobs manager which handles spawning appropriate jobs. pub fn new(spawner: Spawner) -> Self { @@ -566,7 +566,7 @@ where metrics: Job::Metrics, sender: Job::Sender, ) where - Job: JobTrait, + Job: JobTrait, { let hash = leaf.hash; let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -613,7 +613,7 @@ where } /// Send a message to the appropriate job for this `parent_hash`. - async fn send_msg(&mut self, parent_hash: Hash, msg: Consumes) { + async fn send_msg(&mut self, parent_hash: Hash, msg: ToJob) { if let Entry::Occupied(mut job) = self.running.entry(parent_hash) { if job.get_mut().send_msg(msg).await.is_err() { job.remove(); @@ -622,7 +622,7 @@ where } } -impl Stream for Jobs +impl Stream for Jobs where Spawner: SpawnNamed, { @@ -637,7 +637,7 @@ where } } -impl stream::FusedStream for Jobs +impl stream::FusedStream for Jobs where Spawner: SpawnNamed, { @@ -661,7 +661,7 @@ pub struct JobSubsystemParams { /// Conceptually, this is very simple: it just loops forever. /// /// - On incoming overseer messages, it starts or stops jobs as appropriate. -/// - On other incoming messages, if they can be converted into `Job::Consumes` and +/// - On other incoming messages, if they can be converted into `Job::ToJob` and /// include a hash, then they're forwarded to the appropriate individual job. /// - On outgoing messages from the jobs, it forwards them to the overseer. pub struct JobSubsystem { @@ -684,20 +684,20 @@ impl JobSubsystem { where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< - Message = ::Consumes, - OutgoingMessages = ::Outgoing, - Signal = OverseerSignal, + Message = ::ToJob, + OutgoingMessages = ::OutgoingMessages, Sender = ::Sender, + Signal = OverseerSignal, >, Job: 'static + JobTrait + Send, ::RunArgs: Clone + Sync, - ::Consumes: + ::ToJob: Sync + From<::Message>, ::Metrics: Sync, { let JobSubsystem { params: JobSubsystemParams { spawner, run_args, metrics }, .. } = self; - let mut jobs = Jobs::::new(spawner); + let mut jobs = Jobs::::new(spawner); loop { select! { @@ -764,14 +764,14 @@ impl Subsystem for JobSubsystem< where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext< - Message = Job::Consumes, + Message = Job::ToJob, Signal = OverseerSignal, - OutgoingMessages = ::Outgoing, + OutgoingMessages = ::OutgoingMessages, Sender = ::Sender, >, Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - ::Consumes: Sync + From<::Message>, + ::ToJob: Sync + From<::Message>, Job::Metrics: Sync, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index c08d51197766..cb39978549b4 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -44,8 +44,9 @@ use thiserror::Error; // job structs are constructed within JobTrait::run // most will want to retain the sender and receiver, as well as whatever other data they like -struct FakeCollatorProtocolJob { +struct FakeCollatorProtocolJob { receiver: mpsc::Receiver, + _phantom: std::marker::PhantomData, } // Error will mostly be a wrapper to make the try operator more convenient; @@ -57,12 +58,13 @@ enum Error { Sending(#[from] mpsc::SendError), } -impl JobTrait for FakeCollatorProtocolJob +impl JobTrait for FakeCollatorProtocolJob where Sender: overseer::CollatorProtocolSender, { type Consumes = CollatorProtocolMessage; - type Outgoing = CollatorProtocolOutgoingMessages; + type OutgoingMessages = CollatorProtocolOutgoingMessages; + type Sender = Sender; type Error = Error; type RunArgs = bool; type Metrics = (); @@ -72,20 +74,23 @@ where /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run( + fn run( _: ActivatedLeaf, run_args: Self::RunArgs, _metrics: Self::Metrics, receiver: mpsc::Receiver, - mut sender: JobSender, + mut sender: JobSender, ) -> Pin> + Send>> { async move { - let job = FakeCollatorProtocolJob { receiver }; + let job = FakeCollatorProtocolJob { + receiver, + _phantom: std::marker::PhantomData::, + }; if run_args { sender .send_message(CollatorProtocolMessage::Invalid( - Default::default(), + dummy_hash(), dummy_candidate_receipt(dummy_hash()), )) .await; @@ -115,7 +120,13 @@ impl FakeCollatorProtocolJob { } // with the job defined, it's straightforward to get a subsystem implementation. -type FakeCollatorProtocolSubsystem = JobSubsystem; +type FakeCollatorProtocolSubsystem = + JobSubsystem< + FakeCollatorProtocolJob< + test_helpers::TestSubsystemSender + >, + Spawner, + >; // this type lets us pretend to be the overseer type OverseerHandle = test_helpers::TestSubsystemContextHandle; From 50cbe825ccc7beb8bb445b72e8817707f7708886 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 12:50:37 +0200 Subject: [PATCH 047/139] minor fixins --- node/core/approval-voting/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index f713039ee06b..4cda5aca7db7 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -602,11 +602,14 @@ impl State { } /// Bring `session_window` up to date. - pub async fn cache_session_info_for_head( + pub async fn cache_session_info_for_head( &mut self, - ctx: &mut impl overseer::ApprovalVotingContextTrait, + ctx: &mut Context, head: Hash, - ) -> Result, SessionsUnavailable> { + ) -> Result, SessionsUnavailable> + where + Context: overseer::ApprovalVotingContextTrait, + { let session_window = self.session_window.take(); match session_window { None => { From b1e3829761d755cfc932ad61042642e7834359d0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:02:06 +0200 Subject: [PATCH 048/139] split metrics from bridges --- node/network/bridge/src/lib.rs | 203 +------------------------- node/network/bridge/src/metrics.rs | 219 +++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 199 deletions(-) create mode 100644 node/network/bridge/src/metrics.rs diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index a20939612d8e..022b29bd6787 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -33,6 +33,7 @@ use polkadot_node_network_protocol::{ v1 as protocol_v1, ObservedRole, OurView, PeerId, ProtocolVersion, UnifiedReputationChange as Rep, Versioned, View, }; + use polkadot_node_subsystem::{ errors::{SubsystemError, SubsystemResult}, messages::{ @@ -67,6 +68,9 @@ use network::{send_message, Network}; use crate::network::get_peer_id_by_authority_id; +mod metrics; +use self::metrics::Metrics; + #[cfg(test)] mod tests; @@ -83,205 +87,6 @@ const EMPTY_VIEW_COST: Rep = Rep::CostMajor("Peer sent us an empty view"); // network bridge log target const LOG_TARGET: &'static str = "parachain::network-bridge"; -/// Metrics for the network bridge. -#[derive(Clone, Default)] -pub struct Metrics(Option); - -fn peer_set_label(peer_set: PeerSet, version: ProtocolVersion) -> &'static str { - // Higher level code is meant to protect against this ever happening. - peer_set.get_protocol_name_static(version).unwrap_or("") -} - -impl Metrics { - fn on_peer_connected(&self, peer_set: PeerSet, version: ProtocolVersion) { - self.0.as_ref().map(|metrics| { - metrics - .connected_events - .with_label_values(&[peer_set_label(peer_set, version)]) - .inc() - }); - } - - fn on_peer_disconnected(&self, peer_set: PeerSet, version: ProtocolVersion) { - self.0.as_ref().map(|metrics| { - metrics - .disconnected_events - .with_label_values(&[peer_set_label(peer_set, version)]) - .inc() - }); - } - - fn note_peer_count(&self, peer_set: PeerSet, version: ProtocolVersion, count: usize) { - self.0.as_ref().map(|metrics| { - metrics - .peer_count - .with_label_values(&[peer_set_label(peer_set, version)]) - .set(count as u64) - }); - } - - fn on_notification_received(&self, peer_set: PeerSet, version: ProtocolVersion, size: usize) { - if let Some(metrics) = self.0.as_ref() { - metrics - .notifications_received - .with_label_values(&[peer_set_label(peer_set, version)]) - .inc(); - - metrics - .bytes_received - .with_label_values(&[peer_set_label(peer_set, version)]) - .inc_by(size as u64); - } - } - - fn on_notification_sent( - &self, - peer_set: PeerSet, - version: ProtocolVersion, - size: usize, - to_peers: usize, - ) { - if let Some(metrics) = self.0.as_ref() { - metrics - .notifications_sent - .with_label_values(&[peer_set_label(peer_set, version)]) - .inc_by(to_peers as u64); - - metrics - .bytes_sent - .with_label_values(&[peer_set_label(peer_set, version)]) - .inc_by((size * to_peers) as u64); - } - } - - fn note_desired_peer_count(&self, peer_set: PeerSet, size: usize) { - self.0.as_ref().map(|metrics| { - metrics - .desired_peer_count - .with_label_values(&[peer_set.get_default_protocol_name()]) - .set(size as u64) - }); - } - - fn on_report_event(&self) { - if let Some(metrics) = self.0.as_ref() { - metrics.report_events.inc() - } - } -} - -#[derive(Clone)] -struct MetricsInner { - peer_count: prometheus::GaugeVec, - connected_events: prometheus::CounterVec, - disconnected_events: prometheus::CounterVec, - desired_peer_count: prometheus::GaugeVec, - report_events: prometheus::Counter, - - notifications_received: prometheus::CounterVec, - notifications_sent: prometheus::CounterVec, - - bytes_received: prometheus::CounterVec, - bytes_sent: prometheus::CounterVec, -} - -impl metrics::Metrics for Metrics { - fn try_register( - registry: &prometheus::Registry, - ) -> std::result::Result { - let metrics = MetricsInner { - peer_count: prometheus::register( - prometheus::GaugeVec::new( - prometheus::Opts::new( - "polkadot_parachain_peer_count", - "The number of peers on a parachain-related peer-set", - ), - &["protocol"] - )?, - registry, - )?, - connected_events: prometheus::register( - prometheus::CounterVec::new( - prometheus::Opts::new( - "polkadot_parachain_peer_connect_events_total", - "The number of peer connect events on a parachain notifications protocol", - ), - &["protocol"] - )?, - registry, - )?, - disconnected_events: prometheus::register( - prometheus::CounterVec::new( - prometheus::Opts::new( - "polkadot_parachain_peer_disconnect_events_total", - "The number of peer disconnect events on a parachain notifications protocol", - ), - &["protocol"] - )?, - registry, - )?, - desired_peer_count: prometheus::register( - prometheus::GaugeVec::new( - prometheus::Opts::new( - "polkadot_parachain_desired_peer_count", - "The number of peers that the local node is expected to connect to on a parachain-related peer-set (either including or not including unresolvable authorities, depending on whether `ConnectToValidators` or `ConnectToValidatorsResolved` was used.)", - ), - &["protocol"] - )?, - registry, - )?, - report_events: prometheus::register( - prometheus::Counter::new( - "polkadot_parachain_network_report_events_total", - "The amount of reputation changes issued by subsystems", - )?, - registry, - )?, - notifications_received: prometheus::register( - prometheus::CounterVec::new( - prometheus::Opts::new( - "polkadot_parachain_notifications_received_total", - "The number of notifications received on a parachain protocol", - ), - &["protocol"] - )?, - registry, - )?, - notifications_sent: prometheus::register( - prometheus::CounterVec::new( - prometheus::Opts::new( - "polkadot_parachain_notifications_sent_total", - "The number of notifications sent on a parachain protocol", - ), - &["protocol"] - )?, - registry, - )?, - bytes_received: prometheus::register( - prometheus::CounterVec::new( - prometheus::Opts::new( - "polkadot_parachain_notification_bytes_received_total", - "The number of bytes received on a parachain notification protocol", - ), - &["protocol"] - )?, - registry, - )?, - bytes_sent: prometheus::register( - prometheus::CounterVec::new( - prometheus::Opts::new( - "polkadot_parachain_notification_bytes_sent_total", - "The number of bytes sent on a parachain notification protocol", - ), - &["protocol"] - )?, - registry, - )?, - }; - - Ok(Metrics(Some(metrics))) - } -} /// Messages from and to the network. /// diff --git a/node/network/bridge/src/metrics.rs b/node/network/bridge/src/metrics.rs new file mode 100644 index 000000000000..0efc8c01654d --- /dev/null +++ b/node/network/bridge/src/metrics.rs @@ -0,0 +1,219 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_node_subsystem_util::metrics::{self, prometheus}; +use super::{ProtocolVersion, PeerSet}; + + +/// Metrics for the network bridge. +#[derive(Clone, Default)] +pub struct Metrics(pub(crate) Option); + +fn peer_set_label(peer_set: PeerSet, version: ProtocolVersion) -> &'static str { + // Higher level code is meant to protect against this ever happening. + peer_set.get_protocol_name_static(version).unwrap_or("") +} + +impl Metrics { + pub fn on_peer_connected(&self, peer_set: PeerSet, version: ProtocolVersion) { + self.0.as_ref().map(|metrics| { + metrics + .connected_events + .with_label_values(&[peer_set_label(peer_set, version)]) + .inc() + }); + } + + pub fn on_peer_disconnected(&self, peer_set: PeerSet, version: ProtocolVersion) { + self.0.as_ref().map(|metrics| { + metrics + .disconnected_events + .with_label_values(&[peer_set_label(peer_set, version)]) + .inc() + }); + } + + pub fn note_peer_count(&self, peer_set: PeerSet, version: ProtocolVersion, count: usize) { + self.0.as_ref().map(|metrics| { + metrics + .peer_count + .with_label_values(&[peer_set_label(peer_set, version)]) + .set(count as u64) + }); + } + + pub fn on_notification_received(&self, peer_set: PeerSet, version: ProtocolVersion, size: usize) { + if let Some(metrics) = self.0.as_ref() { + metrics + .notifications_received + .with_label_values(&[peer_set_label(peer_set, version)]) + .inc(); + + metrics + .bytes_received + .with_label_values(&[peer_set_label(peer_set, version)]) + .inc_by(size as u64); + } + } + + pub fn on_notification_sent( + &self, + peer_set: PeerSet, + version: ProtocolVersion, + size: usize, + to_peers: usize, + ) { + if let Some(metrics) = self.0.as_ref() { + metrics + .notifications_sent + .with_label_values(&[peer_set_label(peer_set, version)]) + .inc_by(to_peers as u64); + + metrics + .bytes_sent + .with_label_values(&[peer_set_label(peer_set, version)]) + .inc_by((size * to_peers) as u64); + } + } + + pub fn note_desired_peer_count(&self, peer_set: PeerSet, size: usize) { + self.0.as_ref().map(|metrics| { + metrics + .desired_peer_count + .with_label_values(&[peer_set.get_default_protocol_name()]) + .set(size as u64) + }); + } + + pub fn on_report_event(&self) { + if let Some(metrics) = self.0.as_ref() { + metrics.report_events.inc() + } + } +} + +#[derive(Clone)] +struct MetricsInner { + peer_count: prometheus::GaugeVec, + connected_events: prometheus::CounterVec, + disconnected_events: prometheus::CounterVec, + desired_peer_count: prometheus::GaugeVec, + report_events: prometheus::Counter, + + notifications_received: prometheus::CounterVec, + notifications_sent: prometheus::CounterVec, + + bytes_received: prometheus::CounterVec, + bytes_sent: prometheus::CounterVec, +} + +impl metrics::Metrics for Metrics { + fn try_register( + registry: &prometheus::Registry, + ) -> std::result::Result { + let metrics = MetricsInner { + peer_count: prometheus::register( + prometheus::GaugeVec::new( + prometheus::Opts::new( + "polkadot_parachain_peer_count", + "The number of peers on a parachain-related peer-set", + ), + &["protocol"] + )?, + registry, + )?, + connected_events: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_peer_connect_events_total", + "The number of peer connect events on a parachain notifications protocol", + ), + &["protocol"] + )?, + registry, + )?, + disconnected_events: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_peer_disconnect_events_total", + "The number of peer disconnect events on a parachain notifications protocol", + ), + &["protocol"] + )?, + registry, + )?, + desired_peer_count: prometheus::register( + prometheus::GaugeVec::new( + prometheus::Opts::new( + "polkadot_parachain_desired_peer_count", + "The number of peers that the local node is expected to connect to on a parachain-related peer-set (either including or not including unresolvable authorities, depending on whether `ConnectToValidators` or `ConnectToValidatorsResolved` was used.)", + ), + &["protocol"] + )?, + registry, + )?, + report_events: prometheus::register( + prometheus::Counter::new( + "polkadot_parachain_network_report_events_total", + "The amount of reputation changes issued by subsystems", + )?, + registry, + )?, + notifications_received: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_notifications_received_total", + "The number of notifications received on a parachain protocol", + ), + &["protocol"] + )?, + registry, + )?, + notifications_sent: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_notifications_sent_total", + "The number of notifications sent on a parachain protocol", + ), + &["protocol"] + )?, + registry, + )?, + bytes_received: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_notification_bytes_received_total", + "The number of bytes received on a parachain notification protocol", + ), + &["protocol"] + )?, + registry, + )?, + bytes_sent: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_notification_bytes_sent_total", + "The number of bytes sent on a parachain notification protocol", + ), + &["protocol"] + )?, + registry, + )?, + }; + + Ok(Metrics(Some(metrics))) + } +} From 8950b5f84d6e935191a92f44cf6668c88ec7d79a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:27:51 +0200 Subject: [PATCH 049/139] FIXME FIXME TODO XXX networking bridge hacks --- node/network/bridge/src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 022b29bd6787..7eb29be50886 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -1056,9 +1056,10 @@ fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { - for msg in AllMessages::dispatch_iter(event) { - ctx.send_unbounded_message(msg); - } + // XXX FIXME XXX FIXME XXX + // for msg in AllMessages::dispatch_iter(event) { + // ctx.send_unbounded_message(msg); + // } } fn dispatch_collation_event_to_all_unbounded( @@ -1075,7 +1076,8 @@ where I: IntoIterator>, I::IntoIter: Send, { - ctx.send_messages(events.into_iter().flat_map(AllMessages::dispatch_iter)).await + // FIXME XXX XXX TODO XXX FIXME XXX + // ctx.send_messages(events.into_iter().flat_map(AllMessages::dispatch_iter)).await } async fn dispatch_collation_events_to_all(events: I, ctx: &mut impl overseer::NetworkBridgeSenderTrait) From 06555b715b5dcb197976d0e11dee3c3ea737a1ea Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:28:13 +0200 Subject: [PATCH 050/139] fixup approval voting --- node/core/approval-voting/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 4cda5aca7db7..e3b774bb1e9a 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -614,7 +614,7 @@ impl State { match session_window { None => { self.session_window = - Some(RollingSessionWindow::new(ctx.sender(), APPROVAL_SESSIONS, head).await?); + Some(RollingSessionWindow::new::(ctx.sender().clone(), APPROVAL_SESSIONS, head).await?); Ok(None) }, Some(mut session_window) => { From 27c59ab2cdb9938c46b8e18c6c2b37db8094f234 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:28:47 +0200 Subject: [PATCH 051/139] chore --- node/core/pvf-checker/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index 80c4a0d4cd0a..2a731e7a20b1 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -23,7 +23,7 @@ use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUn use polkadot_node_subsystem::{ messages::{CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage, RuntimeApiMessage}, - overseer::{self, PvfCheckerOutgoingMessages}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, + overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, SubsystemResult, SubsystemSender, }; use polkadot_primitives::v2::{ From e199b1545211cfdb49448f13680ab03afffcb7f5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:29:02 +0200 Subject: [PATCH 052/139] fixup approval dist --- node/network/approval-distribution/src/lib.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 560fbdd5e70d..7c8d53f74917 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -419,12 +419,12 @@ impl State { ); { + let mut sender = ctx.sender(); for (peer_id, view) in self.peer_views.iter() { let intersection = view.iter().filter(|h| new_hashes.contains(h)); let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( - ctx, - &self.peer_views, + sender, metrics, &mut self.blocks, &self.topologies, @@ -642,8 +642,7 @@ impl State { } Self::unify_with_peer( - ctx, - &self.peer_views, + ctx.sender(), metrics, &mut self.blocks, &self.topologies, @@ -1203,8 +1202,7 @@ impl State { } async fn unify_with_peer( - ctx: &mut impl overseer::ApprovalDistributionContextTrait, - gossip_peers: &HashSet, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, metrics: &Metrics, entries: &mut HashMap, topologies: &SessionGridTopologies, @@ -1320,7 +1318,7 @@ impl State { "Sending assignments to unified peer", ); - ctx.send_message(NetworkBridgeMessage::SendValidationMessage( + sender.send_message(NetworkBridgeMessage::SendValidationMessage( vec![peer_id.clone()], Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( protocol_v1::ApprovalDistributionMessage::Assignments(assignments_to_send), @@ -1337,7 +1335,7 @@ impl State { "Sending approvals to unified peer", ); - ctx.send_message(NetworkBridgeMessage::SendValidationMessage( + sender.send_message(NetworkBridgeMessage::SendValidationMessage( vec![peer_id], Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( protocol_v1::ApprovalDistributionMessage::Approvals(approvals_to_send), @@ -1634,7 +1632,8 @@ impl ApprovalDistribution { msg: ApprovalDistributionMessage, metrics: &Metrics, rng: &mut (impl CryptoRng + Rng), - ) where + ) + where Context: overseer::ApprovalDistributionContextTrait, { match msg { From d17dabc86b1487f9f1b6addbcd0dab682d627b10 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:29:14 +0200 Subject: [PATCH 053/139] fixup network bridge --- node/network/bridge/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 7eb29be50886..78f54b16085c 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -1089,7 +1089,7 @@ where event .focus() .ok() - .map(|m| AllMessages::CollatorProtocol(CollatorProtocolMessage::NetworkBridgeUpdate(m))) + .map(|m| CollatorProtocolMessage::NetworkBridgeUpdate(m)) }; ctx.send_messages(events.into_iter().flat_map(messages_for)).await From a2a6f193c480d47a6db46ddf4ddd9b132d96b9b0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:29:25 +0200 Subject: [PATCH 054/139] brige: can't leak private type --- node/network/bridge/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/network/bridge/src/metrics.rs b/node/network/bridge/src/metrics.rs index 0efc8c01654d..5df5dc2cb827 100644 --- a/node/network/bridge/src/metrics.rs +++ b/node/network/bridge/src/metrics.rs @@ -106,7 +106,7 @@ impl Metrics { } #[derive(Clone)] -struct MetricsInner { +pub(crate) struct MetricsInner { peer_count: prometheus::GaugeVec, connected_events: prometheus::CounterVec, disconnected_events: prometheus::CounterVec, From dbf595cac575a14657c30be322a5c2490734929b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:29:46 +0200 Subject: [PATCH 055/139] make all Senders send --- node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index cc6e9c3e9ebc..ab45d54f8743 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -382,6 +382,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( #( + #support_crate ::SubsystemSender< #outgoing > )* + + Send }; ts.extend(quote!{ From f39bfb9549b4ec5aa9edf18a5bb3f4e938a00901 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:30:01 +0200 Subject: [PATCH 056/139] overseer annotation --- node/overseer/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 1cd6fed3afb9..0d48539e8d1b 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -501,6 +501,8 @@ pub struct Overseer { BitfieldDistributionMessage, GossipSupportMessage, DisputeDistributionMessage, + CollationGenerationMessage, + CollatorProtocolMessage, ])] network_bridge: NetworkBridge, From 02053ab9fc5bd62a29a3e8be6547cdbb11868b87 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:30:12 +0200 Subject: [PATCH 057/139] rolling session window explicitness --- node/subsystem-util/src/rolling_session_window.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 32698211e157..2f1c5ea1014e 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -94,11 +94,14 @@ pub struct RollingSessionWindow { impl RollingSessionWindow { /// Initialize a new session info cache with the given window size. - pub async fn new( - mut sender: impl overseer::SubsystemSender, + pub async fn new( + mut sender: Sender, window_size: SessionWindowSize, block_hash: Hash, - ) -> Result { + ) -> Result + where + Sender: overseer::SubsystemSender, + { let session_index = get_session_index_for_child(&mut sender, block_hash).await?; let window_start = session_index.saturating_sub(window_size.get() - 1); From 14a23baf671414bf6bacb4e2292195cc29e6cf4a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 18:01:46 +0200 Subject: [PATCH 058/139] chore: fmt --- node/collation-generation/src/lib.rs | 3 +- node/core/approval-voting/src/lib.rs | 84 +++++++++---------- node/core/backing/src/lib.rs | 37 ++++---- node/core/bitfield-signing/src/lib.rs | 8 +- node/core/chain-selection/src/lib.rs | 3 +- node/core/dispute-coordinator/src/lib.rs | 11 ++- .../src/participation/mod.rs | 49 +++++------ .../dispute-coordinator/src/scraping/mod.rs | 16 ++-- node/core/provisioner/src/lib.rs | 38 ++++----- node/core/pvf-checker/src/lib.rs | 4 +- node/network/approval-distribution/src/lib.rs | 38 ++++----- .../src/pov_requester/mod.rs | 5 +- .../src/requester/fetch_task/mod.rs | 15 ++-- .../src/requester/mod.rs | 20 ++--- .../src/requester/session_cache.rs | 5 +- node/network/availability-recovery/src/lib.rs | 42 +++++----- node/network/bitfield-distribution/src/lib.rs | 7 +- node/network/bridge/src/lib.rs | 18 ++-- node/network/bridge/src/metrics.rs | 10 ++- .../src/collator_side/mod.rs | 7 +- .../src/validator_side/mod.rs | 6 +- node/network/dispute-distribution/src/lib.rs | 9 +- .../dispute-distribution/src/receiver/mod.rs | 20 ++--- .../dispute-distribution/src/sender/mod.rs | 16 ++-- .../src/sender/send_task.rs | 6 +- node/network/gossip-support/src/lib.rs | 16 ++-- .../network/statement-distribution/src/lib.rs | 23 +++-- .../proc-macro/src/impl_subsystem.rs | 20 ++--- node/subsystem-util/src/tests.rs | 13 +-- 29 files changed, 245 insertions(+), 304 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index f1f25b02aedb..1d32028a0735 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -391,7 +391,8 @@ async fn handle_new_activations { - self.session_window = - Some(RollingSessionWindow::new::(ctx.sender().clone(), APPROVAL_SESSIONS, head).await?); + self.session_window = Some( + RollingSessionWindow::new::( + ctx.sender().clone(), + APPROVAL_SESSIONS, + head, + ) + .await?, + ); Ok(None) }, Some(mut session_window) => { - let r = - session_window.cache_session_info_for_head(ctx.sender(), head).await.map(Option::Some); + let r = session_window + .cache_session_info_for_head(ctx.sender(), head) + .await + .map(Option::Some); self.session_window = Some(session_window); r }, @@ -2243,14 +2251,12 @@ async fn launch_approval( ); sender - .send_message( - DisputeCoordinatorMessage::IssueLocalStatement( - session_index, - candidate_hash, - candidate.clone(), - false, - ) - ) + .send_message(DisputeCoordinatorMessage::IssueLocalStatement( + session_index, + candidate_hash, + candidate.clone(), + false, + )) .await; metrics_guard.take().on_approval_invalid(); }, @@ -2281,16 +2287,14 @@ async fn launch_approval( let (val_tx, val_rx) = oneshot::channel(); sender - .send_message( - CandidateValidationMessage::ValidateFromExhaustive( - available_data.validation_data, - validation_code, - candidate.clone(), - available_data.pov, - APPROVAL_EXECUTION_TIMEOUT, - val_tx, - ) - ) + .send_message(CandidateValidationMessage::ValidateFromExhaustive( + available_data.validation_data, + validation_code, + candidate.clone(), + available_data.pov, + APPROVAL_EXECUTION_TIMEOUT, + val_tx, + )) .await; match val_rx.await { @@ -2308,14 +2312,12 @@ async fn launch_approval( } else { // Commitments mismatch - issue a dispute. sender - .send_message( - DisputeCoordinatorMessage::IssueLocalStatement( - session_index, - candidate_hash, - candidate.clone(), - false, - ) - ) + .send_message(DisputeCoordinatorMessage::IssueLocalStatement( + session_index, + candidate_hash, + candidate.clone(), + false, + )) .await; metrics_guard.take().on_approval_invalid(); @@ -2332,14 +2334,12 @@ async fn launch_approval( ); sender - .send_message( - DisputeCoordinatorMessage::IssueLocalStatement( - session_index, - candidate_hash, - candidate.clone(), - false, - ) - ) + .send_message(DisputeCoordinatorMessage::IssueLocalStatement( + session_index, + candidate_hash, + candidate.clone(), + false, + )) .await; metrics_guard.take().on_approval_invalid(); @@ -2527,14 +2527,14 @@ where metrics.on_approval_produced(); // dispatch to approval distribution. - ctx.send_unbounded_message( - ApprovalDistributionMessage::DistributeApproval(IndirectSignedApprovalVote { + ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( + IndirectSignedApprovalVote { block_hash, candidate_index: candidate_index as _, validator: validator_index, signature: sig, - }) - ); + }, + )); // dispatch to dispute coordinator. actions.extend(inform_disputes_action); diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 80b76d251712..a3adb322a0d0 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -671,14 +671,12 @@ async fn request_candidate_validation( let (tx, rx) = oneshot::channel(); ctx.sender() - .send_message( - CandidateValidationMessage::ValidateFromChainState( - candidate_receipt, - pov, - BACKING_EXECUTION_TIMEOUT, - tx, - ) - ) + .send_message(CandidateValidationMessage::ValidateFromChainState( + candidate_receipt, + pov, + BACKING_EXECUTION_TIMEOUT, + tx, + )) .await; match rx.await { @@ -723,8 +721,7 @@ async fn validate_and_make_available( PoVData::Ready(pov) => pov, PoVData::FetchFromValidator { from_validator, candidate_hash, pov_hash } => { let _span = span.as_ref().map(|s| s.child("request-pov")); - match request_pov(sender, relay_parent, from_validator, candidate_hash, pov_hash) - .await + match request_pov(sender, relay_parent, from_validator, candidate_hash, pov_hash).await { Err(Error::FetchPoV) => { tx_command @@ -808,8 +805,7 @@ async fn validate_and_make_available( struct ValidatorIndexOutOfBounds; -impl CandidateBackingJob -{ +impl CandidateBackingJob { async fn handle_validated_candidate_command( &mut self, root_span: &jaeger::Span, @@ -1153,14 +1149,15 @@ impl CandidateBackingJob if let (Some(candidate_receipt), Some(dispute_statement)) = (maybe_candidate_receipt, maybe_signed_dispute_statement) { - sender.send_message(DisputeCoordinatorMessage::ImportStatements { - candidate_hash, - candidate_receipt, - session: self.session_index, - statements: vec![(dispute_statement, validator_index)], - pending_confirmation: None, - }) - .await; + sender + .send_message(DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt, + session: self.session_index, + statements: vec![(dispute_statement, validator_index)], + pending_confirmation: None, + }) + .await; } Ok(()) diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index a3b36b6d2b49..3b082de92109 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -202,8 +202,7 @@ where metrics: Self::Metrics, _receiver: mpsc::Receiver, mut sender: JobSender, - ) -> Pin> + Send>> - { + ) -> Pin> + Send>> { let metrics = metrics.clone(); async move { if let LeafStatus::Stale = leaf.status { @@ -293,7 +292,4 @@ where /// `BitfieldSigningSubsystem` manages a number of bitfield signing jobs. pub type BitfieldSigningSubsystem = - JobSubsystem< - BitfieldSigningJob, - Spawner, - >; + JobSubsystem, Spawner>; diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index 9b02c8e1bf07..8e9eaa0a78b0 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -20,7 +20,8 @@ use polkadot_node_primitives::BlockWeight; use polkadot_node_subsystem::{ errors::ChainApiError, messages::{ChainApiMessage, ChainSelectionMessage}, - overseer::{self, SubsystemSender}, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, + overseer::{self, SubsystemSender}, + FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, }; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::v2::{BlockNumber, ConsensusLog, Hash, Header}; diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index ae61cfb455a0..63c1b54acf7a 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -32,8 +32,7 @@ use sc_keystore::LocalKeystore; use polkadot_node_primitives::{CandidateVotes, DISPUTE_WINDOW}; use polkadot_node_subsystem::{ - overseer, - messages::DisputeCoordinatorMessage, ActivatedLeaf, FromOverseer, OverseerSignal, + messages::DisputeCoordinatorMessage, overseer, ActivatedLeaf, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, }; use polkadot_node_subsystem_util::{ @@ -127,7 +126,7 @@ impl Config { impl overseer::Subsystem for DisputeCoordinatorSubsystem where - Context: overseer::DisputeCoordinatorContextTrait, + Context: overseer::DisputeCoordinatorContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async { @@ -260,7 +259,7 @@ impl DisputeCoordinatorSubsystem { ChainScraper, )> where - Context: overseer::DisputeCoordinatorContextTrait, + Context: overseer::DisputeCoordinatorContextTrait, { // Prune obsolete disputes: db::v1::note_current_session(overlay_db, rolling_session_window.latest_session())?; @@ -359,7 +358,7 @@ async fn get_rolling_session_window( ctx: &mut Context, ) -> Result> where - Context: overseer::DisputeCoordinatorContextTrait, + Context: overseer::DisputeCoordinatorContextTrait, { if let Some(leaf) = wait_for_first_leaf(ctx).await? { Ok(Some(( @@ -376,7 +375,7 @@ where /// Wait for `ActiveLeavesUpdate`, returns `None` if `Conclude` signal came first. async fn wait_for_first_leaf(ctx: &mut Context) -> Result> where - Context: overseer::DisputeCoordinatorContextTrait, + Context: overseer::DisputeCoordinatorContextTrait, { loop { match ctx.recv().await? { diff --git a/node/core/dispute-coordinator/src/participation/mod.rs b/node/core/dispute-coordinator/src/participation/mod.rs index c86f79d5886a..5ebdff612748 100644 --- a/node/core/dispute-coordinator/src/participation/mod.rs +++ b/node/core/dispute-coordinator/src/participation/mod.rs @@ -27,9 +27,8 @@ use futures_timer::Delay; use polkadot_node_primitives::{ValidationResult, APPROVAL_EXECUTION_TIMEOUT}; use polkadot_node_subsystem::{ - overseer, messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage, CandidateValidationMessage}, - ActiveLeavesUpdate, RecoveryError, SubsystemContext, SubsystemSender, + overseer, ActiveLeavesUpdate, RecoveryError, SubsystemContext, SubsystemSender, }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; use polkadot_primitives::v2::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex}; @@ -266,14 +265,12 @@ async fn participate( // available data let (recover_available_data_tx, recover_available_data_rx) = oneshot::channel(); sender - .send_message( - AvailabilityRecoveryMessage::RecoverAvailableData( - req.candidate_receipt().clone(), - req.session(), - None, - recover_available_data_tx, - ), - ) + .send_message(AvailabilityRecoveryMessage::RecoverAvailableData( + req.candidate_receipt().clone(), + req.session(), + None, + recover_available_data_tx, + )) .await; let available_data = match recover_available_data_rx.await { @@ -332,14 +329,12 @@ async fn participate( // in the dispute let (store_available_data_tx, store_available_data_rx) = oneshot::channel(); sender - .send_message( - AvailabilityStoreMessage::StoreAvailableData { - candidate_hash: *req.candidate_hash(), - n_validators: req.n_validators() as u32, - available_data: available_data.clone(), - tx: store_available_data_tx, - }, - ) + .send_message(AvailabilityStoreMessage::StoreAvailableData { + candidate_hash: *req.candidate_hash(), + n_validators: req.n_validators() as u32, + available_data: available_data.clone(), + tx: store_available_data_tx, + }) .await; match store_available_data_rx.await { @@ -369,16 +364,14 @@ async fn participate( // same level of leeway. let (validation_tx, validation_rx) = oneshot::channel(); sender - .send_message( - CandidateValidationMessage::ValidateFromExhaustive( - available_data.validation_data, - validation_code, - req.candidate_receipt().clone(), - available_data.pov, - APPROVAL_EXECUTION_TIMEOUT, - validation_tx, - ) - ) + .send_message(CandidateValidationMessage::ValidateFromExhaustive( + available_data.validation_data, + validation_code, + req.candidate_receipt().clone(), + available_data.pov, + APPROVAL_EXECUTION_TIMEOUT, + validation_tx, + )) .await; // we cast votes (either positive or negative) depending on the outcome of diff --git a/node/core/dispute-coordinator/src/scraping/mod.rs b/node/core/dispute-coordinator/src/scraping/mod.rs index 2920f20bc1fb..ece781c39f63 100644 --- a/node/core/dispute-coordinator/src/scraping/mod.rs +++ b/node/core/dispute-coordinator/src/scraping/mod.rs @@ -21,8 +21,8 @@ use lru::LruCache; use polkadot_node_primitives::MAX_FINALITY_LAG; use polkadot_node_subsystem::{ - overseer, - messages::{RuntimeApiMessage, ChainApiMessage}, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError, SubsystemSender, + messages::{ChainApiMessage, RuntimeApiMessage}, + overseer, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError, SubsystemSender, }; use polkadot_node_subsystem_util::runtime::{get_candidate_events, get_on_chain_votes}; use polkadot_primitives::v2::{ @@ -287,13 +287,11 @@ where { let (tx, rx) = oneshot::channel(); sender - .send_message( - ChainApiMessage::Ancestors { - hash: head, - k: num_ancestors as usize, - response_channel: tx, - }, - ) + .send_message(ChainApiMessage::Ancestors { + hash: head, + k: num_ancestors as usize, + response_channel: tx, + }) .await; rx.await diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 4e7cabc84ef5..ff24c3282072 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -170,14 +170,13 @@ where /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - fn run ( + fn run( leaf: ActivatedLeaf, _: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, mut sender: JobSender, - ) -> Pin> + Send>> - { + ) -> Pin> + Send>> { let span = leaf.span.clone(); async move { let job = ProvisionerJob::new(leaf, metrics, receiver); @@ -210,11 +209,7 @@ where } } - async fn run_loop( - mut self, - sender: &mut Sender, - span: PerLeafSpan, - ) -> Result<(), Error> { + async fn run_loop(mut self, sender: &mut Sender, span: PerLeafSpan) -> Result<(), Error> { loop { futures::select! { msg = self.receiver.next() => match msg { @@ -249,7 +244,7 @@ where Ok(()) } - async fn send_inherent_data ( + async fn send_inherent_data( &mut self, sender: &mut Sender, return_senders: Vec>, @@ -526,13 +521,11 @@ async fn select_candidates( // now get the backed candidates corresponding to these candidate receipts let (tx, rx) = oneshot::channel(); sender - .send_message( - CandidateBackingMessage::GetBackedCandidates( - relay_parent, - selected_candidates.clone(), - tx, - ) - ) + .send_message(CandidateBackingMessage::GetBackedCandidates( + relay_parent, + selected_candidates.clone(), + tx, + )) .await; let mut candidates = rx.await.map_err(|err| Error::CanceledBackedCandidates(err))?; @@ -672,9 +665,10 @@ async fn request_votes( ) -> Vec<(SessionIndex, CandidateHash, CandidateVotes)> { let (tx, rx) = oneshot::channel(); // Bounded by block production - `ProvisionerMessage::RequestInherentData`. - sender.send_unbounded_message( - DisputeCoordinatorMessage::QueryCandidateVotes(disputes_to_query, tx) - ); + sender.send_unbounded_message(DisputeCoordinatorMessage::QueryCandidateVotes( + disputes_to_query, + tx, + )); match rx.await { Ok(v) => v, @@ -857,8 +851,4 @@ async fn select_disputes( } /// The provisioner subsystem. -pub type ProvisionerSubsystem = - JobSubsystem< - ProvisionerJob, - Spawner, - >; +pub type ProvisionerSubsystem = JobSubsystem, Spawner>; diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index 2a731e7a20b1..d3c5dd6a4a8c 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -549,9 +549,7 @@ async fn initiate_precheck( let (tx, rx) = oneshot::channel(); sender - .send_message( - CandidateValidationMessage::PreCheck(relay_parent, validation_code_hash, tx) - ) + .send_message(CandidateValidationMessage::PreCheck(relay_parent, validation_code_hash, tx)) .await; let timer = metrics.time_pre_check_judgement(); diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 7c8d53f74917..eda23ac90836 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -1318,13 +1318,14 @@ impl State { "Sending assignments to unified peer", ); - sender.send_message(NetworkBridgeMessage::SendValidationMessage( - vec![peer_id.clone()], - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments_to_send), - )), - )) - .await; + sender + .send_message(NetworkBridgeMessage::SendValidationMessage( + vec![peer_id.clone()], + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(assignments_to_send), + )), + )) + .await; } if !approvals_to_send.is_empty() { @@ -1335,13 +1336,14 @@ impl State { "Sending approvals to unified peer", ); - sender.send_message(NetworkBridgeMessage::SendValidationMessage( - vec![peer_id], - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals_to_send), - )), - )) - .await; + sender + .send_message(NetworkBridgeMessage::SendValidationMessage( + vec![peer_id], + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(approvals_to_send), + )), + )) + .await; } } @@ -1593,9 +1595,8 @@ impl ApprovalDistribution { self, mut ctx: Context, state: &mut State, - rng: &mut (impl CryptoRng + Rng) - ) - where + rng: &mut (impl CryptoRng + Rng), + ) where Context: overseer::ApprovalDistributionContextTrait, { loop { @@ -1632,8 +1633,7 @@ impl ApprovalDistribution { msg: ApprovalDistributionMessage, metrics: &Metrics, rng: &mut (impl CryptoRng + Rng), - ) - where + ) where Context: overseer::ApprovalDistributionContextTrait, { match msg { diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index d7269252e2bf..249d30ff7fd9 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -27,13 +27,10 @@ use polkadot_node_primitives::PoV; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::v2::{AuthorityDiscoveryId, CandidateHash, Hash, ValidatorIndex}; use polkadot_subsystem::{ - overseer, jaeger, messages::{IfDisconnected, NetworkBridgeMessage}, - SubsystemContext, + overseer, SubsystemContext, }; -use polkadot_node_subsystem_util::runtime::RuntimeInfo; -use polkadot_primitives::v2::{AuthorityDiscoveryId, CandidateHash, Hash, ValidatorIndex}; use crate::{ error::{Error, FatalError, JfyiError, Result}, diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index 5620d3ed3aca..2f1bdf5b1d71 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -29,18 +29,13 @@ use polkadot_node_network_protocol::request_response::{ }; use polkadot_node_primitives::ErasureChunk; use polkadot_primitives::v2::{ - AuthorityDiscoveryId, BlakeTwo256, Candi dateHash, GroupIndex, Hash, HashT, OccupiedCore, + AuthorityDiscoveryId, BlakeTwo256, CandidateHash, GroupIndex, Hash, HashT, OccupiedCore, SessionIndex, }; use polkadot_subsystem::{ - overseer, jaeger, messages::{AllMessages, AvailabilityStoreMessage, IfDisconnected, NetworkBridgeMessage}, - SubsystemContext, -}; -use polkadot_primitives::v2::{ - AuthorityDiscoveryId, BlakeTwo256, CandidateHash, GroupIndex, Hash, HashT, OccupiedCore, - SessionIndex, + overseer, SubsystemContext, }; use crate::{ @@ -339,7 +334,8 @@ impl RunningTask { self.sender .send(FromFetchTask::Message( - NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::ImmediateError).into() + NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::ImmediateError) + .into(), )) .await .map_err(|_| TaskError::ShuttingDown)?; @@ -423,7 +419,8 @@ impl RunningTask { candidate_hash: self.request.candidate_hash, chunk, tx, - }.into() + } + .into(), )) .await; if let Err(err) = r { diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index da5913e8e2f6..306af1a6d910 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -32,12 +32,9 @@ use futures::{ Stream, }; -use polkadot_node_subsystem_util::runtime::{get_occupied_cores, RuntimeInfo}; -use polkadot_primitives::v2::{CandidateHash, Hash, OccupiedCore, SessionIndex}; use polkadot_node_subsystem::{ - overseer, - messages::{RuntimeApiMessage, ChainApiMessage}, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, SubsystemContext, + messages::{ChainApiMessage, RuntimeApiMessage}, + overseer, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, SubsystemContext, }; use polkadot_node_subsystem_util::runtime::{get_occupied_cores, RuntimeInfo}; use polkadot_primitives::v2::{CandidateHash, Hash, OccupiedCore, SessionIndex}; @@ -316,12 +313,13 @@ where Sender: overseer::SubsystemSender, { let (tx, rx) = oneshot::channel(); - sender.send_message(ChainApiMessage::Ancestors { - hash: relay_parent, - k: limit, - response_channel: tx, - }) - .await; + sender + .send_message(ChainApiMessage::Ancestors { + hash: relay_parent, + k: limit, + response_channel: tx, + }) + .await; let ancestors = rx .await diff --git a/node/network/availability-distribution/src/requester/session_cache.rs b/node/network/availability-distribution/src/requester/session_cache.rs index 1ff19b9c0c56..69df02ef80a9 100644 --- a/node/network/availability-distribution/src/requester/session_cache.rs +++ b/node/network/availability-distribution/src/requester/session_cache.rs @@ -19,14 +19,11 @@ use std::collections::HashSet; use lru::LruCache; use rand::{seq::SliceRandom, thread_rng}; +use polkadot_node_subsystem::{overseer, SubsystemContext}; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::v2::{ AuthorityDiscoveryId, GroupIndex, Hash, SessionIndex, ValidatorIndex, }; -use polkadot_node_subsystem::{ - SubsystemContext, - overseer, -}; use crate::{ error::{Error, Result}, diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 8e6351780256..0e6ac9b0077e 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -50,7 +50,10 @@ use polkadot_node_primitives::{AvailableData, ErasureChunk}; use polkadot_node_subsystem::{ errors::RecoveryError, jaeger, - messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeMessage, RuntimeApiMessage}, + messages::{ + AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeMessage, + RuntimeApiMessage, + }, overseer::{self, Subsystem}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, SubsystemResult, SubsystemSender, @@ -199,12 +202,10 @@ impl RequestFromBackers { ); sender - .send_message( - NetworkBridgeMessage::SendRequests( - vec![Requests::AvailableDataFetchingV1(req)], - IfDisconnected::ImmediateError, - ) - ) + .send_message(NetworkBridgeMessage::SendRequests( + vec![Requests::AvailableDataFetchingV1(req)], + IfDisconnected::ImmediateError, + )) .await; match response.await { @@ -301,8 +302,7 @@ impl RequestChunksFromValidators { &mut self, params: &RecoveryParams, sender: &mut Sender, - ) - where + ) where Sender: overseer::AvailabilityRecoverySenderTrait, { let num_requests = self.get_desired_request_count(params.threshold); @@ -360,9 +360,10 @@ impl RequestChunksFromValidators { } sender - .send_message( - NetworkBridgeMessage::SendRequests(requests, IfDisconnected::ImmediateError) - ) + .send_message(NetworkBridgeMessage::SendRequests( + requests, + IfDisconnected::ImmediateError, + )) .await; } @@ -499,9 +500,7 @@ impl RequestChunksFromValidators { { let (tx, rx) = oneshot::channel(); sender - .send_message( - AvailabilityStoreMessage::QueryAllChunks(params.candidate_hash, tx) - ) + .send_message(AvailabilityStoreMessage::QueryAllChunks(params.candidate_hash, tx)) .await; match rx.await { @@ -651,16 +650,19 @@ fn reconstructed_data_matches_root( branches.root() == *expected_root } -impl RecoveryTask where Sender: overseer::AvailabilityRecoverySenderTrait { +impl RecoveryTask +where + Sender: overseer::AvailabilityRecoverySenderTrait, +{ async fn run(mut self) -> Result { // First just see if we have the data available locally. { let (tx, rx) = oneshot::channel(); self.sender - .send_message( - AvailabilityStoreMessage::QueryAvailableData(self.params.candidate_hash, tx) - - ) + .send_message(AvailabilityStoreMessage::QueryAvailableData( + self.params.candidate_hash, + tx, + )) .await; match rx.await { diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 67e93f8e8fda..3048a9aa8b79 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -213,7 +213,7 @@ impl BitfieldDistribution { /// Start processing work as passed on from the Overseer. async fn run(self, ctx: Context) where - Context: overseer::BitfieldDistributionContextTrait, + Context: overseer::BitfieldDistributionContextTrait, { let mut state = ProtocolState::default(); let mut rng = rand::rngs::StdRng::from_entropy(); @@ -682,7 +682,7 @@ async fn handle_peer_view_change( origin: PeerId, view: View, rng: &mut (impl CryptoRng + Rng), -)where +) where Context: overseer::BitfieldDistributionContextTrait, { let added = state @@ -741,8 +741,7 @@ async fn send_tracked_gossip_message( dest: PeerId, validator: ValidatorId, message: BitfieldGossipMessage, -) -where +) where Context: overseer::BitfieldDistributionContextTrait, { let job_data = if let Some(job_data) = state.per_relay_parent.get_mut(&message.relay_parent) { diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 78f54b16085c..4884326a589e 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -87,7 +87,6 @@ const EMPTY_VIEW_COST: Rep = Rep::CostMajor("Peer sent us an empty view"); // network bridge log target const LOG_TARGET: &'static str = "parachain::network-bridge"; - /// Messages from and to the network. /// /// As transmitted to and received from subsystems. @@ -1071,8 +1070,10 @@ fn dispatch_collation_event_to_all_unbounded( } } -async fn dispatch_validation_events_to_all(events: I, ctx: &mut impl overseer::NetworkBridgeSenderTrait) -where +async fn dispatch_validation_events_to_all( + events: I, + ctx: &mut impl overseer::NetworkBridgeSenderTrait, +) where I: IntoIterator>, I::IntoIter: Send, { @@ -1080,16 +1081,15 @@ where // ctx.send_messages(events.into_iter().flat_map(AllMessages::dispatch_iter)).await } -async fn dispatch_collation_events_to_all(events: I, ctx: &mut impl overseer::NetworkBridgeSenderTrait) -where +async fn dispatch_collation_events_to_all( + events: I, + ctx: &mut impl overseer::NetworkBridgeSenderTrait, +) where I: IntoIterator>, I::IntoIter: Send, { let messages_for = |event: NetworkBridgeEvent| { - event - .focus() - .ok() - .map(|m| CollatorProtocolMessage::NetworkBridgeUpdate(m)) + event.focus().ok().map(|m| CollatorProtocolMessage::NetworkBridgeUpdate(m)) }; ctx.send_messages(events.into_iter().flat_map(messages_for)).await diff --git a/node/network/bridge/src/metrics.rs b/node/network/bridge/src/metrics.rs index 5df5dc2cb827..b648a01a9248 100644 --- a/node/network/bridge/src/metrics.rs +++ b/node/network/bridge/src/metrics.rs @@ -14,9 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use super::{PeerSet, ProtocolVersion}; use polkadot_node_subsystem_util::metrics::{self, prometheus}; -use super::{ProtocolVersion, PeerSet}; - /// Metrics for the network bridge. #[derive(Clone, Default)] @@ -55,7 +54,12 @@ impl Metrics { }); } - pub fn on_notification_received(&self, peer_set: PeerSet, version: ProtocolVersion, size: usize) { + pub fn on_notification_received( + &self, + peer_set: PeerSet, + version: ProtocolVersion, + size: usize, + ) { if let Some(metrics) = self.0.as_ref() { metrics .notifications_received diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 03bc3526e2ec..4cc4c09afa27 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -38,7 +38,9 @@ use polkadot_node_network_protocol::{ use polkadot_node_primitives::{CollationSecondedSignal, PoV, Statement}; use polkadot_node_subsystem::{ jaeger, - messages::{RuntimeApiMessage, CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage}, + messages::{ + CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, RuntimeApiMessage, + }, overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, }; use polkadot_node_subsystem_util::{ @@ -464,8 +466,7 @@ async fn determine_core( sender: &mut impl overseer::SubsystemSender, para_id: ParaId, relay_parent: Hash, -) -> Result> -{ +) -> Result> { let cores = get_availability_cores(sender, relay_parent).await?; for (idx, core) in cores.iter().enumerate() { diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index 401973b24696..2656ec128e5c 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -47,9 +47,8 @@ use polkadot_node_primitives::{PoV, SignedFullStatement}; use polkadot_node_subsystem::{ jaeger, messages::{ - RuntimeApiMessage, CandidateBackingMessage, CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, - NetworkBridgeMessage, + NetworkBridgeMessage, RuntimeApiMessage, }, overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, SubsystemSender, }; @@ -690,8 +689,7 @@ async fn report_collator( ctx: &mut Context, peer_data: &HashMap, id: CollatorId, -) -where +) where Context: overseer::CollatorProtocolContextTrait, { if let Some(peer_id) = collator_peer_id(peer_data, &id) { diff --git a/node/network/dispute-distribution/src/lib.rs b/node/network/dispute-distribution/src/lib.rs index 24467f40b59b..6b710921af68 100644 --- a/node/network/dispute-distribution/src/lib.rs +++ b/node/network/dispute-distribution/src/lib.rs @@ -116,9 +116,7 @@ pub struct DisputeDistributionSubsystem { impl overseer::Subsystem for DisputeDistributionSubsystem where - Context: overseer::DisputeDistributionContextTrait - + Sync - + Send, + Context: overseer::DisputeDistributionContextTrait + Sync + Send, AD: AuthorityDiscovery + Clone, { fn start(self, ctx: Context) -> SpawnedSubsystem { @@ -207,7 +205,7 @@ where signal: OverseerSignal, ) -> Result where - Context: overseer::DisputeDistributionContextTrait, + Context: overseer::DisputeDistributionContextTrait, { match signal { OverseerSignal::Conclude => return Ok(SignalResult::Conclude), @@ -225,7 +223,8 @@ where ctx: &mut Context, msg: DisputeDistributionMessage, ) -> Result<()> - where Context: overseer::DisputeDistributionContextTrait, + where + Context: overseer::DisputeDistributionContextTrait, { match msg { DisputeDistributionMessage::SendDispute(dispute_msg) => diff --git a/node/network/dispute-distribution/src/receiver/mod.rs b/node/network/dispute-distribution/src/receiver/mod.rs index ecc36ac8a2da..f0d9cc8262eb 100644 --- a/node/network/dispute-distribution/src/receiver/mod.rs +++ b/node/network/dispute-distribution/src/receiver/mod.rs @@ -39,11 +39,9 @@ use polkadot_node_network_protocol::{ PeerId, UnifiedReputationChange as Rep, }; use polkadot_node_primitives::DISPUTE_WINDOW; -use polkadot_node_subsystem_util::{runtime, runtime::RuntimeInfo}; use polkadot_node_subsystem::{ - overseer, messages::{AllMessages, DisputeCoordinatorMessage, ImportStatementsResult}, - SubsystemSender, + overseer, SubsystemSender, }; use polkadot_node_subsystem_util::{runtime, runtime::RuntimeInfo}; @@ -268,15 +266,13 @@ where let (pending_confirmation, confirmation_rx) = oneshot::channel(); let candidate_hash = candidate_receipt.hash(); self.sender - .send_message( - DisputeCoordinatorMessage::ImportStatements { - candidate_hash, - candidate_receipt, - session: valid_vote.0.session_index(), - statements: vec![valid_vote, invalid_vote], - pending_confirmation: Some(pending_confirmation), - } - ) + .send_message(DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt, + session: valid_vote.0.session_index(), + statements: vec![valid_vote, invalid_vote], + pending_confirmation: Some(pending_confirmation), + }) .await; self.pending_imports.push(peer, confirmation_rx, pending_response); diff --git a/node/network/dispute-distribution/src/sender/mod.rs b/node/network/dispute-distribution/src/sender/mod.rs index bf559c2390b8..74816cacc31d 100644 --- a/node/network/dispute-distribution/src/sender/mod.rs +++ b/node/network/dispute-distribution/src/sender/mod.rs @@ -20,12 +20,9 @@ use futures::channel::{mpsc, oneshot}; use polkadot_node_network_protocol::request_response::v1::DisputeRequest; use polkadot_node_primitives::{CandidateVotes, DisputeMessage, SignedDisputeStatement}; -use polkadot_node_subsystem_util::runtime::RuntimeInfo; -use polkadot_primitives::v2::{CandidateHash, DisputeStatement, Hash, SessionIndex}; use polkadot_node_subsystem::{ - overseer, messages::{AllMessages, DisputeCoordinatorMessage}, - ActiveLeavesUpdate, SubsystemContext, + overseer, ActiveLeavesUpdate, SubsystemContext, }; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::v2::{CandidateHash, DisputeStatement, Hash, SessionIndex}; @@ -363,9 +360,7 @@ where let (tx, rx) = oneshot::channel(); // Caller scope is in `update_leaves` and this is bounded by fork count. - ctx.send_unbounded_message( - DisputeCoordinatorMessage::ActiveDisputes(tx), - ); + ctx.send_unbounded_message(DisputeCoordinatorMessage::ActiveDisputes(tx)); rx.await.map_err(|_| JfyiError::AskActiveDisputesCanceled) } @@ -380,9 +375,10 @@ where { let (tx, rx) = oneshot::channel(); // Caller scope is in `update_leaves` and this is bounded by fork count. - ctx.send_unbounded_message( - DisputeCoordinatorMessage::QueryCandidateVotes(vec![(session_index, candidate_hash)], tx), - ); + ctx.send_unbounded_message(DisputeCoordinatorMessage::QueryCandidateVotes( + vec![(session_index, candidate_hash)], + tx, + )); rx.await .map(|v| v.get(0).map(|inner| inner.to_owned().2)) .map_err(|_| JfyiError::AskCandidateVotesCanceled) diff --git a/node/network/dispute-distribution/src/sender/send_task.rs b/node/network/dispute-distribution/src/sender/send_task.rs index f1457527a523..8f0fe5453629 100644 --- a/node/network/dispute-distribution/src/sender/send_task.rs +++ b/node/network/dispute-distribution/src/sender/send_task.rs @@ -26,11 +26,7 @@ use polkadot_node_network_protocol::{ }, IfDisconnected, }; -use polkadot_node_subsystem::{ - overseer, - messages::NetworkBridgeMessage, - SubsystemContext, -}; +use polkadot_node_subsystem::{messages::NetworkBridgeMessage, overseer, SubsystemContext}; use polkadot_node_subsystem_util::{metrics, runtime::RuntimeInfo}; use polkadot_primitives::v2::{ AuthorityDiscoveryId, CandidateHash, Hash, SessionIndex, ValidatorIndex, diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index e43301fbe7e7..7f0244615d1b 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -315,8 +315,7 @@ where &mut self, sender: &mut Sender, authorities: Vec, - ) - where + ) where Sender: overseer::GossipSupportSenderTrait, { let num = authorities.len(); @@ -541,12 +540,13 @@ async fn update_gossip_topology( .map(|i| (authorities[i].clone(), ValidatorIndex::from(i as u32))) .collect(); - sender.send_message(NetworkBridgeMessage::NewGossipTopology { - session: session_index, - our_neighbors_x: row_neighbors, - our_neighbors_y: column_neighbors, - }) - .await; + sender + .send_message(NetworkBridgeMessage::NewGossipTopology { + session: session_index, + our_neighbors_x: row_neighbors, + our_neighbors_y: column_neighbors, + }) + .await; Ok(()) } diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 7d9031275db5..cc0de5b75988 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -1118,11 +1118,8 @@ async fn send_statements_about( statement = ?statement.statement, "Sending statement", ); - ctx.send_message(NetworkBridgeMessage::SendValidationMessage( - vec![peer.clone()], - payload, - )) - .await; + ctx.send_message(NetworkBridgeMessage::SendValidationMessage(vec![peer.clone()], payload)) + .await; metrics.on_statement_distributed(); } @@ -1152,19 +1149,19 @@ async fn send_statements( statement = ?statement.statement, "Sending statement" ); - ctx.send_message(NetworkBridgeMessage::SendValidationMessage( - vec![peer.clone()], - payload, - )) - .await; + ctx.send_message(NetworkBridgeMessage::SendValidationMessage(vec![peer.clone()], payload)) + .await; metrics.on_statement_distributed(); } } -async fn report_peer(ctx: &mut impl overseer::StatementDistributionContextTrait, peer: PeerId, rep: Rep) { - ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)) - .await +async fn report_peer( + ctx: &mut impl overseer::StatementDistributionContextTrait, + peer: PeerId, + rep: Rep, +) { + ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await } /// If message contains a statement, then retrieve it, otherwise fork task to fetch it. diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index ab45d54f8743..f0af3ff3d801 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -164,8 +164,8 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let subsystem_name = ssf.generic.to_string(); let outgoing_wrapper = &Ident::new(&(subsystem_name.clone() + "OutgoingMessages"), span); - let subsystem_ctx_trait= &Ident::new(&(subsystem_name.clone() + "ContextTrait"), span); - let subsystem_sender_trait= &Ident::new(&(subsystem_name.clone() + "SenderTrait"), span); + let subsystem_ctx_trait = &Ident::new(&(subsystem_name.clone() + "ContextTrait"), span); + let subsystem_sender_trait = &Ident::new(&(subsystem_name.clone() + "SenderTrait"), span); ts.extend(impl_per_subsystem_helper_traits( info, @@ -174,7 +174,8 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { subsystem_sender_trait, &ssf.message_to_consume, &ssf.messages_to_send, - outgoing_wrapper)); + outgoing_wrapper, + )); ts.extend(impl_associate_outgoing_messages(&ssf.message_to_consume, &outgoing_wrapper)); @@ -358,7 +359,6 @@ pub(crate) fn impl_associate_outgoing_messages( } } - pub(crate) fn impl_per_subsystem_helper_traits( info: &OverseerInfo, subsystem_ctx_trait: &Ident, @@ -377,7 +377,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( // Create a helper trait bound of all outgoing messages, and the generated wrapper type // for ease of use within subsystems: - let acc_sender_trait_bounds = quote !{ + let acc_sender_trait_bounds = quote! { #support_crate ::SubsystemSender< #outgoing_wrapper > #( + #support_crate ::SubsystemSender< #outgoing > @@ -385,7 +385,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( + Send }; - ts.extend(quote!{ + ts.extend(quote! { /// A abstracting trait for usage with subsystems. pub trait #subsystem_sender_trait : #acc_sender_trait_bounds {} @@ -396,16 +396,15 @@ pub(crate) fn impl_per_subsystem_helper_traits( {} }); - // Create a helper accumulated per subsystem trait bound: - let where_clause = quote !{ + let where_clause = quote! { #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, #all_messages_wrapper: From< #outgoing_wrapper >, #all_messages_wrapper: From< #consumes >, Self::Sender: #subsystem_sender_trait + #acc_sender_trait_bounds, }; - ts.extend(quote!{ + ts.extend(quote! { /// Accumuative trait for a particular subsystem wrapper. pub trait #subsystem_ctx_trait : SubsystemContext < Message = #consumes, @@ -433,7 +432,6 @@ pub(crate) fn impl_per_subsystem_helper_traits( {} }); - // // impl the subsystem context trait (alt!) // ts.extend(quote!{ @@ -627,7 +625,7 @@ pub(crate) fn impl_subsystem_context( }; // impl the subsystem context trait - ts.extend(quote!{ + ts.extend(quote! { #[#support_crate ::async_trait] impl #support_crate ::SubsystemContext for #subsystem_ctx_name diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index cb39978549b4..180cb384e2ca 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -82,10 +82,8 @@ where mut sender: JobSender, ) -> Pin> + Send>> { async move { - let job = FakeCollatorProtocolJob { - receiver, - _phantom: std::marker::PhantomData::, - }; + let job = + FakeCollatorProtocolJob { receiver, _phantom: std::marker::PhantomData:: }; if run_args { sender @@ -121,12 +119,7 @@ impl FakeCollatorProtocolJob { // with the job defined, it's straightforward to get a subsystem implementation. type FakeCollatorProtocolSubsystem = - JobSubsystem< - FakeCollatorProtocolJob< - test_helpers::TestSubsystemSender - >, - Spawner, - >; + JobSubsystem, Spawner>; // this type lets us pretend to be the overseer type OverseerHandle = test_helpers::TestSubsystemContextHandle; From d6ef62dae998c112e6302afa96cf43e3122bbb8d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:48:44 +0200 Subject: [PATCH 059/139] fixins --- node/core/approval-voting/src/lib.rs | 1 - node/core/dispute-coordinator/src/lib.rs | 4 ++-- node/core/dispute-coordinator/src/scraping/mod.rs | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 3a55acd4d256..9a5b63461669 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -876,7 +876,6 @@ async fn handle_actions( Action::ScheduleWakeup { block_hash, block_number, candidate_hash, tick } => wakeups.schedule(block_hash, block_number, candidate_hash, tick), Action::IssueApproval(candidate_hash, approval_request) => { - let mut sender = ctx.sender().clone(); // Note that the IssueApproval action will create additional // actions that will need to all be processed before we can // handle the next action in the set passed to the ambient diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 63c1b54acf7a..4186ab5129fe 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -32,8 +32,8 @@ use sc_keystore::LocalKeystore; use polkadot_node_primitives::{CandidateVotes, DISPUTE_WINDOW}; use polkadot_node_subsystem::{ - messages::DisputeCoordinatorMessage, overseer, ActivatedLeaf, FromOverseer, OverseerSignal, - SpawnedSubsystem, SubsystemContext, SubsystemError, + overseer, ActivatedLeaf, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, + SubsystemError, }; use polkadot_node_subsystem_util::{ database::Database, rolling_session_window::RollingSessionWindow, diff --git a/node/core/dispute-coordinator/src/scraping/mod.rs b/node/core/dispute-coordinator/src/scraping/mod.rs index ece781c39f63..6fcae077ad8a 100644 --- a/node/core/dispute-coordinator/src/scraping/mod.rs +++ b/node/core/dispute-coordinator/src/scraping/mod.rs @@ -21,8 +21,8 @@ use lru::LruCache; use polkadot_node_primitives::MAX_FINALITY_LAG; use polkadot_node_subsystem::{ - messages::{ChainApiMessage, RuntimeApiMessage}, - overseer, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError, SubsystemSender, + messages::ChainApiMessage, overseer, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError, + SubsystemSender, }; use polkadot_node_subsystem_util::runtime::{get_candidate_events, get_on_chain_votes}; use polkadot_primitives::v2::{ From fb161d480a74234e988d7bb8550b1cfd640ab0c7 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 27 Apr 2022 13:59:32 +0200 Subject: [PATCH 060/139] minor gimps --- node/core/approval-voting/src/import.rs | 1 + node/core/backing/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index fc6981c7da68..90070c78ed16 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -45,6 +45,7 @@ use polkadot_node_subsystem_util::{ rolling_session_window::{RollingSessionWindow, SessionWindowUpdate}, }; use polkadot_primitives::v2::{ + Id as ParaId, BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, ConsensusLog, CoreIndex, GroupIndex, Hash, Header, SessionIndex, }; diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index a3adb322a0d0..d0e15e9e412c 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -37,7 +37,7 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::{ jaeger, messages::{ - AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage, + AvailabilityDistributionMessage, AvailabilityStoreMessage, CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, DisputeCoordinatorMessage, ProvisionableData, ProvisionerMessage, RuntimeApiRequest, StatementDistributionMessage, From daf5f6e88f3a3e095d9722ff692a4524bacd11b4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 28 Apr 2022 11:47:58 +0200 Subject: [PATCH 061/139] avoid impl Trait, use explicit generics --- node/core/approval-voting/src/import.rs | 13 ++-- node/core/approval-voting/src/lib.rs | 12 +-- node/core/av-store/src/lib.rs | 2 +- .../dispute-coordinator/src/initialized.rs | 24 +++--- .../src/participation/tests.rs | 12 +-- node/network/approval-distribution/src/lib.rs | 61 ++++++++------- node/network/bridge/src/lib.rs | 6 +- .../src/validator_side/mod.rs | 4 +- .../network/statement-distribution/src/lib.rs | 75 ++++++++++--------- 9 files changed, 111 insertions(+), 98 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index 90070c78ed16..eb4a1f84c9ca 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -45,9 +45,8 @@ use polkadot_node_subsystem_util::{ rolling_session_window::{RollingSessionWindow, SessionWindowUpdate}, }; use polkadot_primitives::v2::{ - Id as ParaId, BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, ConsensusLog, CoreIndex, - GroupIndex, Hash, Header, SessionIndex, + GroupIndex, Hash, Header, Id as ParaId, SessionIndex, }; use sc_keystore::LocalKeystore; use sp_consensus_slots::Slot; @@ -108,8 +107,8 @@ enum ImportedBlockInfoError { } /// Computes information about the imported block. Returns an error if the info couldn't be extracted. -async fn imported_block_info( - ctx: &mut impl overseer::ApprovalVotingContextTrait, +async fn imported_block_info( + ctx: &mut Context, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -320,10 +319,8 @@ pub struct BlockImportedCandidates { /// * and return information about all candidates imported under each block. /// /// It is the responsibility of the caller to schedule wakeups for each block. -pub(crate) async fn handle_new_head( - ctx: &mut impl overseer::ApprovalVotingContextTrait< - OutgoingMessages = overseer::ApprovalVotingOutgoingMessages, - >, +pub(crate) async fn handle_new_head( + ctx: &mut Context, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, head: Hash, diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 9a5b63461669..406ce5f958bf 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -857,8 +857,8 @@ where // https://github.com/paritytech/polkadot/issues/3311 // // returns `true` if any of the actions was a `Conclude` command. -async fn handle_actions( - ctx: &mut impl overseer::ApprovalVotingContextTrait, +async fn handle_actions( + ctx: &mut Context, state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, @@ -1093,8 +1093,8 @@ fn distribution_messages_for_activation( } // Handle an incoming signal from the overseer. Returns true if execution should conclude. -async fn handle_from_overseer( - ctx: &mut impl overseer::ApprovalVotingContextTrait, +async fn handle_from_overseer( + ctx: &mut Context, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, metrics: &Metrics, @@ -2156,8 +2156,8 @@ fn process_wakeup( // Launch approval work, returning an `AbortHandle` which corresponds to the background task // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. -async fn launch_approval( - ctx: &mut impl overseer::ApprovalVotingContextTrait, +async fn launch_approval( + ctx: &mut Context, metrics: Metrics, session_index: SessionIndex, candidate: CandidateReceipt, diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 50bb5ab9f8fa..5c801db2f19e 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -524,7 +524,7 @@ where Context: overseer::AvailabilityStoreContextTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { - let future = run(self, ctx).map(|_| Ok(())).boxed(); + let future = run::(self, ctx).map(|_| Ok(())).boxed(); SpawnedSubsystem { name: "availability-store-subsystem", future } } diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index c41eda4b8f80..013b8771a8f3 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -251,9 +251,9 @@ impl Initialized { } } - async fn process_active_leaves_update( + async fn process_active_leaves_update( &mut self, - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, + ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, update: ActiveLeavesUpdate, now: u64, @@ -315,9 +315,9 @@ impl Initialized { /// Scrapes on-chain votes (backing votes and concluded disputes) for a active leaf of the /// relay chain. - async fn process_on_chain_votes( + async fn process_on_chain_votes( &mut self, - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, + ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, votes: ScrapedOnChainVotes, now: u64, @@ -493,9 +493,9 @@ impl Initialized { Ok(()) } - async fn handle_incoming( + async fn handle_incoming( &mut self, - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, + ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, message: DisputeCoordinatorMessage, now: Timestamp, @@ -630,9 +630,9 @@ impl Initialized { Ok(()) } - async fn handle_import_statements( + async fn handle_import_statements( &mut self, - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, + ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, candidate_receipt: MaybeCandidateReceipt, @@ -919,9 +919,9 @@ impl Initialized { Ok(ImportStatementsResult::ValidImport) } - async fn issue_local_statement( + async fn issue_local_statement( &mut self, - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, + ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, candidate_receipt: CandidateReceipt, @@ -1051,8 +1051,8 @@ enum MuxedMessage { } impl MuxedMessage { - async fn receive( - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, + async fn receive( + ctx: &mut Context, from_sender: &mut participation::WorkerMessageReceiver, ) -> FatalResult { // We are only fusing here to make `select` happy, in reality we will quit if the stream diff --git a/node/core/dispute-coordinator/src/participation/tests.rs b/node/core/dispute-coordinator/src/participation/tests.rs index b6c104cc34ec..44e583c86ea5 100644 --- a/node/core/dispute-coordinator/src/participation/tests.rs +++ b/node/core/dispute-coordinator/src/participation/tests.rs @@ -53,16 +53,16 @@ pub fn make_our_subsystem_context( make_subsystem_context(spawn) } -async fn participate( - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, +async fn participate( + ctx: &mut Context, participation: &mut Participation, ) -> Result<()> { let commitments = CandidateCommitments::default(); participate_with_commitments_hash(ctx, participation, commitments.hash()).await } -async fn participate_with_commitments_hash( - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, +async fn participate_with_commitments_hash( + ctx: &mut Context, participation: &mut Participation, commitments_hash: Hash, ) -> Result<()> { @@ -81,8 +81,8 @@ async fn participate_with_commitments_hash( .await } -async fn activate_leaf( - ctx: &mut impl overseer::DisputeCoordinatorContextTrait, +async fn activate_leaf( + ctx: &mut Context, participation: &mut Participation, block_number: BlockNumber, ) -> FatalResult<()> { diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index eda23ac90836..34d19f129497 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -322,9 +322,9 @@ enum PendingMessage { } impl State { - async fn handle_network_msg( + async fn handle_network_msg( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), @@ -376,9 +376,9 @@ impl State { } } - async fn handle_new_blocks( + async fn handle_new_blocks( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), @@ -495,9 +495,9 @@ impl State { self.enable_aggression(ctx, Resend::Yes, metrics).await; } - async fn handle_new_session_topology( + async fn handle_new_session_topology( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, session: SessionIndex, topology: SessionGridTopology, ) { @@ -518,13 +518,16 @@ impl State { .await; } - async fn process_incoming_peer_message( + async fn process_incoming_peer_message< + Context: overseer::ApprovalDistributionContextTrait, + R: CryptoRng + Rng, + >( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, msg: protocol_v1::ApprovalDistributionMessage, - rng: &mut (impl CryptoRng + Rng), + rng: &mut R, ) { match msg { protocol_v1::ApprovalDistributionMessage::Assignments(assignments) => { @@ -609,13 +612,16 @@ impl State { // handle a peer view change: requires that the peer is already connected // and has an entry in the `PeerData` struct. - async fn handle_peer_view_change( + async fn handle_peer_view_change< + Context: overseer::ApprovalDistributionContextTrait, + R: CryptoRng + Rng, + >( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, view: View, - rng: &mut (impl CryptoRng + Rng), + rng: &mut R, ) { gum::trace!(target: LOG_TARGET, ?view, "Peer view change"); let finalized_number = view.finalized_number; @@ -654,9 +660,9 @@ impl State { .await; } - async fn handle_block_finalized( + async fn handle_block_finalized( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, finalized_number: BlockNumber, ) { @@ -682,14 +688,17 @@ impl State { self.enable_aggression(ctx, Resend::No, metrics).await; } - async fn import_and_circulate_assignment( + async fn import_and_circulate_assignment< + Context: overseer::ApprovalDistributionContextTrait, + R: CyrptoRng + Rng, + >( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCert, claimed_candidate_index: CandidateIndex, - rng: &mut (impl CryptoRng + Rng), + rng: &mut R, ) { let block_hash = assignment.block_hash.clone(); let validator_index = assignment.validator; @@ -940,9 +949,9 @@ impl State { } } - async fn import_and_circulate_approval( + async fn import_and_circulate_approval( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, @@ -1347,9 +1356,9 @@ impl State { } } - async fn enable_aggression( + async fn enable_aggression( &mut self, - ctx: &mut impl overseer::ApprovalDistributionContextTrait, + ctx: &mut Context, resend: Resend, metrics: &Metrics, ) { @@ -1450,8 +1459,10 @@ impl State { // // Note that the required routing of a message can be modified even if the // topology is unknown yet. -async fn adjust_required_routing_and_propagate( - ctx: &mut impl overseer::ApprovalDistributionContextTrait, +async fn adjust_required_routing_and_propagate< + Context: overseer::ApprovalDistributionContextTrait, +>( + ctx: &mut Context, blocks: &mut HashMap, topologies: &SessionGridTopologies, block_filter: impl Fn(&mut BlockEntry) -> bool, @@ -1557,8 +1568,8 @@ async fn adjust_required_routing_and_propagate( } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation( - ctx: &mut impl overseer::ApprovalDistributionContextTrait, +async fn modify_reputation( + ctx: &mut Context, peer_id: PeerId, rep: Rep, ) { diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 4884326a589e..e609577399bf 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -906,9 +906,9 @@ fn construct_view( View::new(live_heads.take(MAX_VIEW_HEADS), finalized_number) } -fn update_our_view( - net: &mut impl Network, - ctx: &mut impl overseer::NetworkBridgeContextTrait, +fn update_our_view( + net: &mut Net, + ctx: &mut Context, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index 2656ec128e5c..f48398585a6c 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -1294,8 +1294,8 @@ async fn poll_requests( } /// Dequeue another collation and fetch. -async fn dequeue_next_collation_and_fetch( - ctx: &mut impl overseer::CollatorProtocolContextTrait, +async fn dequeue_next_collation_and_fetch( + ctx: &mut Context, state: &mut State, relay_parent: Hash, // The collator we tried to fetch from last. diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index cc0de5b75988..0ff27cdde525 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -630,8 +630,8 @@ enum MuxedMessage { } impl MuxedMessage { - async fn receive( - ctx: &mut impl overseer::StatementDistributionContextTrait, + async fn receive( + ctx: &mut Context, from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, ) -> MuxedMessage { @@ -888,11 +888,13 @@ fn check_statement_signature( /// circulates the statement to all peers who have not seen it yet, and /// sends all statements dependent on that statement to peers who could previously not receive /// them but now can. -async fn circulate_statement_and_dependents( +async fn circulate_statement_and_dependents< + Context: overseer::StatementDistributionContextTrait, +>( gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, relay_parent: Hash, statement: SignedFullStatement, priority_peers: Vec, @@ -1004,10 +1006,10 @@ fn is_statement_large(statement: &SignedFullStatement) -> (bool, Option) /// Circulates a statement to all peers who have not seen it yet, and returns /// an iterator over peers who need to have dependent statements sent. -async fn circulate_statement<'a>( +async fn circulate_statement<'a, Context: overseer::StatementDistributionContextTrait>( gossip_peers: &HashSet, peers: &mut HashMap, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, relay_parent: Hash, stored: StoredStatement<'a>, mut priority_peers: Vec, @@ -1093,10 +1095,10 @@ async fn circulate_statement<'a>( } /// Send all statements about a given candidate hash to a peer. -async fn send_statements_about( +async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, relay_parent: Hash, candidate_hash: CandidateHash, active_head: &ActiveHeadData, @@ -1126,10 +1128,10 @@ async fn send_statements_about( } /// Send all statements at a given relay-parent to a peer. -async fn send_statements( +async fn send_statements( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, @@ -1156,8 +1158,8 @@ async fn send_statements( } } -async fn report_peer( - ctx: &mut impl overseer::StatementDistributionContextTrait, +async fn report_peer( + ctx: &mut Context, peer: PeerId, rep: Rep, ) { @@ -1171,11 +1173,14 @@ async fn report_peer( /// your statement. /// /// If the message was large, but the result has been fetched already that one is returned. -async fn retrieve_statement_from_message<'a>( +async fn retrieve_statement_from_message< + 'a, + Context: overseer::StatementDistributionContextTrait, +>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, active_head: &'a mut ActiveHeadData, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option { @@ -1263,11 +1268,11 @@ async fn retrieve_statement_from_message<'a>( /// Launch request for a large statement and get tracking status. /// /// Returns `None` if spawning task failed. -async fn launch_request( +async fn launch_request( meta: StatementMetadata, peer: PeerId, req_sender: mpsc::Sender, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, metrics: &Metrics, ) -> Option { let (task, handle) = @@ -1294,13 +1299,16 @@ async fn launch_request( /// Handle incoming message and circulate it to peers, if we did not know it already. /// -async fn handle_incoming_message_and_circulate<'a>( +async fn handle_incoming_message_and_circulate< + 'a, + Context: overseer::StatementDistributionContextTrait, +>( peer: PeerId, gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &'a mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1351,12 +1359,12 @@ async fn handle_incoming_message_and_circulate<'a>( // // This function checks the signature and ensures the statement is compatible with our // view. It also notifies candidate backing if the statement was previously unknown. -async fn handle_incoming_message<'a>( +async fn handle_incoming_message<'a, Context: overseer::StatementDistributionContextTrait>( peer: PeerId, peer_data: &mut PeerData, active_heads: &'a mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1548,11 +1556,13 @@ async fn handle_incoming_message<'a>( } /// Update a peer's view. Sends all newly unlocked statements based on the previous -async fn update_peer_view_and_maybe_send_unlocked( +async fn update_peer_view_and_maybe_send_unlocked< + Context: overseer::StatementDistributionContextTrait, +>( peer: PeerId, gossip_peers: &HashSet, peer_data: &mut PeerData, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, active_heads: &HashMap, new_view: View, metrics: &Metrics, @@ -1587,13 +1597,13 @@ async fn update_peer_view_and_maybe_send_unlocked( } } -async fn handle_network_update( +async fn handle_network_update( peers: &mut HashMap, gossip_peers: &mut HashSet, authorities: &mut HashMap, active_heads: &mut HashMap, recent_outdated_heads: &RecentOutdatedHeads, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, req_sender: &mpsc::Sender, update: NetworkBridgeEvent, metrics: &Metrics, @@ -1703,14 +1713,9 @@ impl StatementDistributionSubsystem { Self { keystore, req_receiver: Some(req_receiver), metrics, rng } } - async fn run( + async fn run( mut self, - mut ctx: impl overseer::StatementDistributionContextTrait< - Message = StatementDistributionMessage, - OutgoingMessages = overseer::StatementDistributionOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - >, + mut ctx: Context, ) -> std::result::Result<(), FatalError> { let mut peers: HashMap = HashMap::new(); let mut gossip_peers: HashSet = HashSet::new(); @@ -1827,9 +1832,9 @@ impl StatementDistributionSubsystem { Ok(()) } - async fn handle_requester_message( + async fn handle_requester_message( &mut self, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &mut HashMap, @@ -1935,9 +1940,9 @@ impl StatementDistributionSubsystem { Ok(()) } - async fn handle_subsystem_message( + async fn handle_subsystem_message( &mut self, - ctx: &mut impl overseer::StatementDistributionContextTrait, + ctx: &mut Context, runtime: &mut RuntimeInfo, peers: &mut HashMap, gossip_peers: &mut HashSet, From 93513305a8e64ccd49e5593fc12eb16223d59e8e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 28 Apr 2022 11:51:25 +0200 Subject: [PATCH 062/139] disable type Sender = overseer::OverseerSender Required to enable tests, doesn't compile because of some rust limitation not deducing the requirements for associated types (here: Sender ) properly. --- .../proc-macro/src/impl_subsystem.rs | 18 +++++++++++------- .../overseer-gen/proc-macro/src/lib.rs | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index f0af3ff3d801..1ea2d505bf6d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -401,7 +401,8 @@ pub(crate) fn impl_per_subsystem_helper_traits( #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, #all_messages_wrapper: From< #outgoing_wrapper >, #all_messages_wrapper: From< #consumes >, - Self::Sender: #subsystem_sender_trait + #acc_sender_trait_bounds, + ::Sender: #subsystem_sender_trait, + ::Sender: #acc_sender_trait_bounds, }; ts.extend(quote! { @@ -410,13 +411,15 @@ pub(crate) fn impl_per_subsystem_helper_traits( Message = #consumes, Signal = #signal, OutgoingMessages = #outgoing_wrapper, - // FIXME setting sender breaks testing, which wants to override this - Sender = #subsystem_sender_name < #outgoing_wrapper > , + // Sender, Error = #error_ty, > where #where_clause - {} + { + /// Sender. + type Sender: #subsystem_sender_trait + #acc_sender_trait_bounds; + } impl #subsystem_ctx_trait for T where @@ -424,12 +427,13 @@ pub(crate) fn impl_per_subsystem_helper_traits( Message = #consumes, Signal = #signal, OutgoingMessages = #outgoing_wrapper, - // FIXME setting sender breaks testing - Sender = #subsystem_sender_name < #outgoing_wrapper >, + // Sender Error = #error_ty, >, #where_clause - {} + { + type Sender = ::Sender; + } }); // // impl the subsystem context trait (alt!) diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index d87a391c6890..e198a0152d7f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -99,7 +99,7 @@ pub(crate) fn impl_overseer_gen( let ts = expander::Expander::new("overlord-expansion") .add_comment("Generated overseer code by `#[overlord(..)]`".to_owned()) .dry(!cfg!(feature = "expand")) - .verbose(false) + .verbose(true) .fmt(expander::Edition::_2021) .write_to_out_dir(additive) .expect("Expander does not fail due to IO in OUT_DIR. qed"); From bb69d2a563af091cb517f6e734a9223137a4124e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 28 Apr 2022 11:52:16 +0200 Subject: [PATCH 063/139] chore: silence some warnings --- node/subsystem-util/src/determine_new_blocks.rs | 2 +- node/subsystem-util/src/lib.rs | 2 +- node/subsystem-util/src/rolling_session_window.rs | 2 +- node/subsystem-util/src/runtime/mod.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index 0a330d7f04fc..bccc81f5ed97 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -18,7 +18,7 @@ use futures::{channel::oneshot, prelude::*}; use polkadot_node_subsystem::{ - messages::{ChainApiMessage, RuntimeApiMessage}, + messages::ChainApiMessage, SubsystemSender, }; use polkadot_primitives::v2::{BlockNumber, Hash, Header}; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 63ba4d04ccd9..46318c20c771 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -27,7 +27,7 @@ use polkadot_node_subsystem::{ errors::{RuntimeApiError, SubsystemError}, messages::{ - AllMessages, BoundToRelayParent, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, + BoundToRelayParent, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, }, overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemSender, diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 2f1c5ea1014e..d8a908a6901c 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -26,7 +26,7 @@ use futures::channel::oneshot; use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{RuntimeApiMessage, RuntimeApiRequest}, - overseer, SubsystemContext, + overseer, }; use thiserror::Error; diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 47bdb1e97f89..0028116901d4 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -26,7 +26,7 @@ use sp_core::crypto::ByteArray; use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; use polkadot_node_subsystem::{ - messages::RuntimeApiMessage, overseer, SubsystemContext, SubsystemSender, + messages::RuntimeApiMessage, overseer, SubsystemSender, }; use polkadot_primitives::v2::{ CandidateEvent, CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, From 592acd399502ba739503e4551a9ddb2cdf4d309a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 28 Apr 2022 12:27:54 +0200 Subject: [PATCH 064/139] add additional `::Sender` bounds Please rustc, since it can't resolve the associated trait bounds as I'd anticipated. --- node/collation-generation/src/lib.rs | 15 +- node/core/approval-voting/src/import.rs | 18 ++- node/core/approval-voting/src/lib.rs | 30 +++- node/core/av-store/src/lib.rs | 12 ++ node/core/candidate-validation/src/lib.rs | 4 + node/core/chain-api/src/lib.rs | 2 + node/core/chain-selection/src/lib.rs | 3 + .../dispute-coordinator/src/initialized.rs | 58 +++++-- node/core/dispute-coordinator/src/lib.rs | 12 ++ .../src/participation/mod.rs | 32 +++- .../src/participation/tests.rs | 29 ++-- node/core/pvf-checker/src/lib.rs | 1 + node/core/runtime-api/src/lib.rs | 2 + node/network/approval-distribution/src/lib.rs | 119 +++++++++----- .../availability-distribution/src/lib.rs | 4 + .../src/pov_requester/mod.rs | 2 + .../src/requester/fetch_task/mod.rs | 2 + .../src/requester/mod.rs | 6 + .../src/requester/session_cache.rs | 4 + node/network/availability-recovery/src/lib.rs | 10 ++ node/network/bitfield-distribution/src/lib.rs | 20 +++ node/network/bridge/src/lib.rs | 11 +- .../src/collator_side/mod.rs | 22 +++ node/network/collator-protocol/src/lib.rs | 6 + .../src/validator_side/mod.rs | 34 +++- node/network/dispute-distribution/src/lib.rs | 14 +- .../dispute-distribution/src/sender/mod.rs | 14 ++ .../src/sender/send_task.rs | 37 ++++- node/network/gossip-support/src/lib.rs | 3 + .../network/statement-distribution/src/lib.rs | 153 ++++++++++++------ .../src/determine_new_blocks.rs | 5 +- node/subsystem-util/src/lib.rs | 4 +- node/subsystem-util/src/runtime/mod.rs | 4 +- 33 files changed, 542 insertions(+), 150 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index 1d32028a0735..a61b6b98bcbc 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -74,6 +74,8 @@ impl CollationGenerationSubsystem { async fn run(mut self, mut ctx: Context) where Context: overseer::CollationGenerationContextTrait, + ::Sender: + overseer::CollationGenerationSenderTrait, { // when we activate new leaves, we spawn a bunch of sub-tasks, each of which is // expected to generate precisely one message. We don't want to block the main loop @@ -111,6 +113,8 @@ impl CollationGenerationSubsystem { ) -> bool where Context: overseer::CollationGenerationContextTrait, + ::Sender: + overseer::CollationGenerationSenderTrait, { match incoming { Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { @@ -163,6 +167,8 @@ impl CollationGenerationSubsystem { impl overseer::Subsystem for CollationGenerationSubsystem where Context: overseer::CollationGenerationContextTrait, + ::Sender: + overseer::CollationGenerationSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { @@ -175,13 +181,18 @@ where } } -async fn handle_new_activations( +async fn handle_new_activations( config: Arc, activated: impl IntoIterator, ctx: &mut Context, metrics: Metrics, sender: &mpsc::Sender, -) -> crate::error::Result<()> { +) -> crate::error::Result<()> +where + Context: overseer::CollationGenerationContextTrait, + ::Sender: + overseer::CollationGenerationSenderTrait, +{ // follow the procedure from the guide: // https://w3f.github.io/parachain-implementers-guide/node/collators/collation-generation.html diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index eb4a1f84c9ca..b3ae5d8b4de2 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -107,12 +107,16 @@ enum ImportedBlockInfoError { } /// Computes information about the imported block. Returns an error if the info couldn't be extracted. -async fn imported_block_info( +async fn imported_block_info( ctx: &mut Context, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, -) -> Result { +) -> Result +where + Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, +{ // Ignore any runtime API errors - that means these blocks are old and finalized. // Only unfinalized blocks factor into the approval voting process. @@ -319,13 +323,17 @@ pub struct BlockImportedCandidates { /// * and return information about all candidates imported under each block. /// /// It is the responsibility of the caller to schedule wakeups for each block. -pub(crate) async fn handle_new_head( +pub(crate) async fn handle_new_head( ctx: &mut Context, state: &mut State, - db: &mut OverlayedBackend<'_, impl Backend>, + db: &mut OverlayedBackend<'_, B>, head: Hash, finalized_number: &Option, -) -> SubsystemResult> { +) -> SubsystemResult> +where + Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, +{ const MAX_HEADS_LOOK_BACK: BlockNumber = MAX_FINALITY_LAG; let mut span = jaeger::Span::new(head, "approval-checking-import"); diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 406ce5f958bf..7329950faedd 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -358,6 +358,7 @@ impl ApprovalVotingSubsystem { impl overseer::Subsystem for ApprovalVotingSubsystem where Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); @@ -609,6 +610,8 @@ impl State { ) -> Result, SessionsUnavailable> where Context: overseer::ApprovalVotingContextTrait, + ::Sender: + overseer::ApprovalVotingSenderTrait, { let session_window = self.session_window.take(); match session_window { @@ -720,6 +723,7 @@ async fn run( ) -> SubsystemResult<()> where Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, B: Backend, { let mut state = State { @@ -857,7 +861,7 @@ where // https://github.com/paritytech/polkadot/issues/3311 // // returns `true` if any of the actions was a `Conclude` command. -async fn handle_actions( +async fn handle_actions( ctx: &mut Context, state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, @@ -867,7 +871,11 @@ async fn handle_actions( approvals_cache: &mut lru::LruCache, mode: &mut Mode, actions: Vec, -) -> SubsystemResult { +) -> SubsystemResult +where + Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, +{ let mut conclude = false; let mut actions_iter = actions.into_iter(); @@ -1093,7 +1101,7 @@ fn distribution_messages_for_activation( } // Handle an incoming signal from the overseer. Returns true if execution should conclude. -async fn handle_from_overseer( +async fn handle_from_overseer( ctx: &mut Context, state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, @@ -1101,7 +1109,11 @@ async fn handle_from_overseer( x: FromOverseer, last_finalized_height: &mut Option, wakeups: &mut Wakeups, -) -> SubsystemResult> { +) -> SubsystemResult> +where + Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, +{ let actions = match x { FromOverseer::Signal(OverseerSignal::ActiveLeaves(update)) => { let mut actions = Vec::new(); @@ -1212,6 +1224,7 @@ async fn handle_approved_ancestor( ) -> SubsystemResult> where Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, { const MAX_TRACING_WINDOW: usize = 200; const ABNORMAL_DEPTH_THRESHOLD: usize = 5; @@ -2156,7 +2169,7 @@ fn process_wakeup( // Launch approval work, returning an `AbortHandle` which corresponds to the background task // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. -async fn launch_approval( +async fn launch_approval( ctx: &mut Context, metrics: Metrics, session_index: SessionIndex, @@ -2164,7 +2177,11 @@ async fn launch_approval( validator_index: ValidatorIndex, block_hash: Hash, backing_group: GroupIndex, -) -> SubsystemResult> { +) -> SubsystemResult> +where + Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, +{ let (a_tx, a_rx) = oneshot::channel(); let (code_tx, code_rx) = oneshot::channel(); @@ -2374,6 +2391,7 @@ async fn issue_approval( ) -> SubsystemResult> where Context: overseer::ApprovalVotingContextTrait, + ::Sender: overseer::ApprovalVotingSenderTrait, { let block_entry = match db.load_block_entry(&block_hash)? { Some(b) => b, diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 5c801db2f19e..af6a4f2df5a5 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -522,6 +522,8 @@ impl KnownUnfinalizedBlocks { impl overseer::Subsystem for AvailabilityStoreSubsystem where Context: overseer::AvailabilityStoreContextTrait, + ::Sender: + overseer::AvailabilityStoreSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(self, ctx).map(|_| Ok(())).boxed(); @@ -533,6 +535,8 @@ where async fn run(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context) where Context: overseer::AvailabilityStoreContextTrait, + ::Sender: + overseer::AvailabilityStoreSenderTrait, { let mut next_pruning = Delay::new(subsystem.pruning_config.pruning_interval).fuse(); @@ -561,6 +565,8 @@ async fn run_iteration( ) -> Result where Context: overseer::AvailabilityStoreContextTrait, + ::Sender: + overseer::AvailabilityStoreSenderTrait, { select! { incoming = ctx.recv().fuse() => { @@ -612,6 +618,8 @@ async fn process_block_activated( ) -> Result<(), Error> where Context: overseer::AvailabilityStoreContextTrait, + ::Sender: + overseer::AvailabilityStoreSenderTrait, { let now = subsystem.clock.now()?; @@ -671,6 +679,8 @@ async fn process_new_head( ) -> Result<(), Error> where Context: overseer::AvailabilityStoreContextTrait, + ::Sender: + overseer::AvailabilityStoreSenderTrait, { let candidate_events = util::request_candidate_events(hash, ctx.sender()).await.await??; @@ -817,6 +827,8 @@ async fn process_block_finalized( ) -> Result<(), Error> where Context: overseer::AvailabilityStoreContextTrait, + ::Sender: + overseer::AvailabilityStoreSenderTrait, { let now = subsystem.clock.now()?; diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 2094821365dc..da72e7b3b803 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -96,6 +96,8 @@ impl CandidateValidationSubsystem { impl overseer::Subsystem for CandidateValidationSubsystem where Context: overseer::CandidateValidationContextTrait, + ::Sender: + overseer::CandidateValidationSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run( @@ -120,6 +122,8 @@ async fn run( ) -> SubsystemResult<()> where Context: overseer::CandidateValidationContextTrait, + ::Sender: + overseer::CandidateValidationSenderTrait, { let (validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 7e4240c93517..4929ae50b4bb 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -68,6 +68,7 @@ impl overseer::Subsystem for ChainApiS where Client: HeaderBackend + AuxStore + 'static, Context: overseer::ChainApiContextTrait, + ::Sender: overseer::ChainApiSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(ctx, self) @@ -84,6 +85,7 @@ async fn run( where Client: HeaderBackend + AuxStore, Context: overseer::ChainApiContextTrait, + ::Sender: overseer::ChainApiSenderTrait, { loop { match ctx.recv().await? { diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index 8e9eaa0a78b0..1383d3286db7 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -332,6 +332,7 @@ impl ChainSelectionSubsystem { impl overseer::Subsystem for ChainSelectionSubsystem where Context: overseer::ChainSelectionContextTrait, + ::Sender: overseer::ChainSelectionSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = db_backend::v1::DbBackend::new( @@ -355,6 +356,7 @@ async fn run( clock: Box, ) where Context: overseer::ChainSelectionContextTrait, + ::Sender: overseer::ChainSelectionSenderTrait, B: Backend, { loop { @@ -386,6 +388,7 @@ async fn run_until_error( ) -> Result<(), Error> where Context: overseer::ChainSelectionContextTrait, + ::Sender: overseer::ChainSelectionSenderTrait, B: Backend, { let mut stagnant_check_stream = stagnant_check_interval.timeout_stream(); diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index 013b8771a8f3..d38e1b5f36f7 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -124,6 +124,8 @@ impl Initialized { ) -> FatalResult<()> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, B: Backend, { loop { @@ -161,6 +163,8 @@ impl Initialized { ) -> Result<()> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, B: Backend, { for (priority, request) in participations.drain(..) { @@ -251,13 +255,18 @@ impl Initialized { } } - async fn process_active_leaves_update( + async fn process_active_leaves_update( &mut self, ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, update: ActiveLeavesUpdate, now: u64, - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { let on_chain_votes = self.scraper.process_active_leaves_update(ctx.sender(), &update).await?; self.participation.process_active_leaves_update(ctx, &update).await?; @@ -315,13 +324,18 @@ impl Initialized { /// Scrapes on-chain votes (backing votes and concluded disputes) for a active leaf of the /// relay chain. - async fn process_on_chain_votes( + async fn process_on_chain_votes( &mut self, ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, votes: ScrapedOnChainVotes, now: u64, - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { let ScrapedOnChainVotes { session, backing_validators_per_candidate, disputes } = votes; if backing_validators_per_candidate.is_empty() && disputes.is_empty() { @@ -493,13 +507,18 @@ impl Initialized { Ok(()) } - async fn handle_incoming( + async fn handle_incoming( &mut self, ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, message: DisputeCoordinatorMessage, now: Timestamp, - ) -> Result JfyiResult<()>>> { + ) -> Result JfyiResult<()>>> + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { match message { DisputeCoordinatorMessage::ImportStatements { candidate_hash, @@ -630,7 +649,7 @@ impl Initialized { Ok(()) } - async fn handle_import_statements( + async fn handle_import_statements( &mut self, ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, @@ -639,7 +658,12 @@ impl Initialized { session: SessionIndex, statements: Vec<(SignedDisputeStatement, ValidatorIndex)>, now: Timestamp, - ) -> Result { + ) -> Result + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { if session + DISPUTE_WINDOW.get() < self.highest_session { // It is not valid to participate in an ancient dispute (spam?). return Ok(ImportStatementsResult::InvalidImport) @@ -919,7 +943,7 @@ impl Initialized { Ok(ImportStatementsResult::ValidImport) } - async fn issue_local_statement( + async fn issue_local_statement( &mut self, ctx: &mut Context, overlay_db: &mut OverlayedBackend<'_, impl Backend>, @@ -928,7 +952,12 @@ impl Initialized { session: SessionIndex, valid: bool, now: Timestamp, - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { // Load session info. let info = match self.rolling_session_window.session_info(session) { None => { @@ -1051,10 +1080,15 @@ enum MuxedMessage { } impl MuxedMessage { - async fn receive( + async fn receive( ctx: &mut Context, from_sender: &mut participation::WorkerMessageReceiver, - ) -> FatalResult { + ) -> FatalResult + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { // We are only fusing here to make `select` happy, in reality we will quit if the stream // ends. let from_overseer = ctx.recv().fuse(); diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 4186ab5129fe..8d371c572d42 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -127,6 +127,8 @@ impl Config { impl overseer::Subsystem for DisputeCoordinatorSubsystem where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async { @@ -161,6 +163,8 @@ impl DisputeCoordinatorSubsystem { ) -> FatalResult<()> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, B: Backend + 'static, { let res = self.initialize(&mut ctx, backend, &*clock).await?; @@ -193,6 +197,8 @@ impl DisputeCoordinatorSubsystem { > where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, B: Backend + 'static, { loop { @@ -260,6 +266,8 @@ impl DisputeCoordinatorSubsystem { )> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, { // Prune obsolete disputes: db::v1::note_current_session(overlay_db, rolling_session_window.latest_session())?; @@ -359,6 +367,8 @@ async fn get_rolling_session_window( ) -> Result> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, { if let Some(leaf) = wait_for_first_leaf(ctx).await? { Ok(Some(( @@ -376,6 +386,8 @@ where async fn wait_for_first_leaf(ctx: &mut Context) -> Result> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, { loop { match ctx.recv().await? { diff --git a/node/core/dispute-coordinator/src/participation/mod.rs b/node/core/dispute-coordinator/src/participation/mod.rs index 5ebdff612748..37008e598126 100644 --- a/node/core/dispute-coordinator/src/participation/mod.rs +++ b/node/core/dispute-coordinator/src/participation/mod.rs @@ -152,6 +152,8 @@ impl Participation { ) -> Result<()> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, { // Participation already running - we can ignore that request: if self.running_participations.contains(req.candidate_hash()) { @@ -177,11 +179,16 @@ impl Participation { /// /// Returns: The received `ParticipationStatement` or a fatal error, in case /// something went wrong when dequeuing more requests (tasks could not be spawned). - pub async fn get_participation_result( + pub async fn get_participation_result( &mut self, ctx: &mut Context, msg: WorkerMessage, - ) -> FatalResult { + ) -> FatalResult + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { let WorkerMessage(statement) = msg; self.running_participations.remove(&statement.candidate_hash); let recent_block = self.recent_block.expect("We never ever reset recent_block to `None` and we already received a result, so it must have been set before. qed."); @@ -193,11 +200,16 @@ impl Participation { /// /// Make sure we to dequeue participations if that became possible and update most recent /// block. - pub async fn process_active_leaves_update( + pub async fn process_active_leaves_update( &mut self, ctx: &mut Context, update: &ActiveLeavesUpdate, - ) -> FatalResult<()> { + ) -> FatalResult<()> + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { if let Some(activated) = &update.activated { match self.recent_block { None => { @@ -215,11 +227,17 @@ impl Participation { } /// Dequeue until `MAX_PARALLEL_PARTICIPATIONS` is reached. - async fn dequeue_until_capacity( + + async fn dequeue_until_capacity( &mut self, ctx: &mut Context, recent_head: Hash, - ) -> FatalResult<()> { + ) -> FatalResult<()> + where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, + { while self.running_participations.len() < MAX_PARALLEL_PARTICIPATIONS { if let Some(req) = self.queue.dequeue() { self.fork_participation(ctx, req, recent_head)?; @@ -239,6 +257,8 @@ impl Participation { ) -> FatalResult<()> where Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, { if self.running_participations.insert(req.candidate_hash().clone()) { let sender = ctx.sender().clone(); diff --git a/node/core/dispute-coordinator/src/participation/tests.rs b/node/core/dispute-coordinator/src/participation/tests.rs index 44e583c86ea5..68962af2023f 100644 --- a/node/core/dispute-coordinator/src/participation/tests.rs +++ b/node/core/dispute-coordinator/src/participation/tests.rs @@ -52,20 +52,26 @@ pub fn make_our_subsystem_context( ) { make_subsystem_context(spawn) } - -async fn participate( - ctx: &mut Context, - participation: &mut Participation, -) -> Result<()> { +async fn participate(ctx: &mut Context, participation: &mut Participation) -> Result<()> +where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, +{ let commitments = CandidateCommitments::default(); participate_with_commitments_hash(ctx, participation, commitments.hash()).await } -async fn participate_with_commitments_hash( +async fn participate_with_commitments_hash( ctx: &mut Context, participation: &mut Participation, commitments_hash: Hash, -) -> Result<()> { +) -> Result<()> +where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, +{ let candidate_receipt = { let mut receipt = dummy_candidate_receipt_bad_sig(dummy_hash(), dummy_hash()); receipt.commitments_hash = commitments_hash; @@ -81,11 +87,16 @@ async fn participate_with_commitments_hash( +async fn activate_leaf( ctx: &mut Context, participation: &mut Participation, block_number: BlockNumber, -) -> FatalResult<()> { +) -> FatalResult<()> +where + Context: overseer::DisputeCoordinatorContextTrait, + ::Sender: + overseer::DisputeCoordinatorSenderTrait, +{ let block_header = Header { parent_hash: BlakeTwo256::hash(&block_number.encode()), number: block_number, diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index d3c5dd6a4a8c..6951d485e909 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -63,6 +63,7 @@ impl PvfCheckerSubsystem { impl overseer::Subsystem for PvfCheckerSubsystem where Context: overseer::PvfCheckerContextTrait, + ::Sender: overseer::PvfCheckerSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { if self.enabled { diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 1a662a1383f8..3275273db940 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -97,6 +97,7 @@ where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, Context: overseer::RuntimeApiContextTrait, + ::Sender: overseer::RuntimeApiSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { future: run(ctx, self).boxed(), name: "runtime-api-subsystem" } @@ -340,6 +341,7 @@ where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, Context: overseer::RuntimeApiContextTrait, + ::Sender: overseer::RuntimeApiSenderTrait, { loop { select! { diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 34d19f129497..3e399932cf4f 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -322,13 +322,17 @@ enum PendingMessage { } impl State { - async fn handle_network_msg( + async fn handle_network_msg( &mut self, ctx: &mut Context, metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + { match event { NetworkBridgeEvent::PeerConnected(peer_id, role, _, _) => { // insert a blank view if none already present @@ -376,13 +380,17 @@ impl State { } } - async fn handle_new_blocks( + async fn handle_new_blocks( &mut self, ctx: &mut Context, metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + { let mut new_hashes = HashSet::new(); for meta in &metas { match self.blocks.entry(meta.hash.clone()) { @@ -495,12 +503,16 @@ impl State { self.enable_aggression(ctx, Resend::Yes, metrics).await; } - async fn handle_new_session_topology( + async fn handle_new_session_topology( &mut self, ctx: &mut Context, session: SessionIndex, topology: SessionGridTopology, - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + { self.topologies.insert_topology(session, topology); let topology = self.topologies.get_topology(session).expect("just inserted above; qed"); @@ -518,17 +530,19 @@ impl State { .await; } - async fn process_incoming_peer_message< - Context: overseer::ApprovalDistributionContextTrait, - R: CryptoRng + Rng, - >( + async fn process_incoming_peer_message( &mut self, ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, msg: protocol_v1::ApprovalDistributionMessage, rng: &mut R, - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + R: CryptoRng + Rng, + { match msg { protocol_v1::ApprovalDistributionMessage::Assignments(assignments) => { gum::trace!( @@ -612,17 +626,19 @@ impl State { // handle a peer view change: requires that the peer is already connected // and has an entry in the `PeerData` struct. - async fn handle_peer_view_change< - Context: overseer::ApprovalDistributionContextTrait, - R: CryptoRng + Rng, - >( + async fn handle_peer_view_change( &mut self, ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, view: View, rng: &mut R, - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + R: CryptoRng + Rng, + { gum::trace!(target: LOG_TARGET, ?view, "Peer view change"); let finalized_number = view.finalized_number; let old_view = @@ -660,12 +676,16 @@ impl State { .await; } - async fn handle_block_finalized( + async fn handle_block_finalized( &mut self, ctx: &mut Context, metrics: &Metrics, finalized_number: BlockNumber, - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + { // we want to prune every block up to (including) finalized_number // why +1 here? // split_off returns everything after the given key, including the key @@ -688,10 +708,7 @@ impl State { self.enable_aggression(ctx, Resend::No, metrics).await; } - async fn import_and_circulate_assignment< - Context: overseer::ApprovalDistributionContextTrait, - R: CyrptoRng + Rng, - >( + async fn import_and_circulate_assignment( &mut self, ctx: &mut Context, metrics: &Metrics, @@ -699,7 +716,12 @@ impl State { assignment: IndirectAssignmentCert, claimed_candidate_index: CandidateIndex, rng: &mut R, - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + R: CyrptoRng + Rng, + { let block_hash = assignment.block_hash.clone(); let validator_index = assignment.validator; @@ -949,13 +971,17 @@ impl State { } } - async fn import_and_circulate_approval( + async fn import_and_circulate_approval( &mut self, ctx: &mut Context, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + { let block_hash = vote.block_hash.clone(); let validator_index = vote.validator; let candidate_index = vote.candidate_index; @@ -1356,12 +1382,16 @@ impl State { } } - async fn enable_aggression( + async fn enable_aggression( &mut self, ctx: &mut Context, resend: Resend, metrics: &Metrics, - ) { + ) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + { let min_age = self.blocks_by_number.iter().next().map(|(num, _)| num); let max_age = self.blocks_by_number.iter().rev().next().map(|(num, _)| num); let config = self.aggression_config.clone(); @@ -1459,15 +1489,19 @@ impl State { // // Note that the required routing of a message can be modified even if the // topology is unknown yet. -async fn adjust_required_routing_and_propagate< - Context: overseer::ApprovalDistributionContextTrait, ->( +async fn adjust_required_routing_and_propagate( ctx: &mut Context, blocks: &mut HashMap, topologies: &SessionGridTopologies, - block_filter: impl Fn(&mut BlockEntry) -> bool, - routing_modifier: impl Fn(&mut RequiredRouting, bool, &ValidatorIndex), -) { + block_filter: BlockFilter, + routing_modifier: RoutingModifier, +) where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, + BlockFilter: Fn(&mut BlockEntry) -> bool, + RoutingModifier: Fn(&mut RequiredRouting, bool, &ValidatorIndex), +{ let mut peer_assignments = HashMap::new(); let mut peer_approvals = HashMap::new(); @@ -1568,11 +1602,12 @@ async fn adjust_required_routing_and_propagate< } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation( - ctx: &mut Context, - peer_id: PeerId, - rep: Rep, -) { +async fn modify_reputation(ctx: &mut Context, peer_id: PeerId, rep: Rep) +where + Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, +{ gum::trace!( target: LOG_TARGET, reputation = ?rep, @@ -1592,6 +1627,8 @@ impl ApprovalDistribution { async fn run(self, ctx: Context) where Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, { let mut state = State::default(); @@ -1609,6 +1646,8 @@ impl ApprovalDistribution { rng: &mut (impl CryptoRng + Rng), ) where Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, { loop { let message = match ctx.recv().await { @@ -1646,6 +1685,8 @@ impl ApprovalDistribution { rng: &mut (impl CryptoRng + Rng), ) where Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, { match msg { ApprovalDistributionMessage::NetworkBridgeUpdate(event) => { @@ -1692,6 +1733,8 @@ impl ApprovalDistribution { impl overseer::Subsystem for ApprovalDistribution where Context: overseer::ApprovalDistributionContextTrait, + ::Sender: + overseer::ApprovalDistributionSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index 964117b043e4..0bd5ead794bb 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -71,6 +71,8 @@ pub struct IncomingRequestReceivers { impl overseer::Subsystem for AvailabilityDistributionSubsystem where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -97,6 +99,8 @@ impl AvailabilityDistributionSubsystem { async fn run(self, mut ctx: Context) -> std::result::Result<(), FatalError> where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { let Self { mut runtime, recvs, metrics } = self; diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index 249d30ff7fd9..ce9e755fba11 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -51,6 +51,8 @@ pub async fn fetch_pov( ) -> Result<()> where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { let info = &runtime.get_session_info(ctx.sender(), parent).await?.session_info; let authority_id = info diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index 2f1bdf5b1d71..0f092151f25c 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -178,6 +178,8 @@ impl FetchTask { pub async fn start(config: FetchTaskConfig, ctx: &mut Context) -> Result where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { let FetchTaskConfig { prepared_running, live_in } = config; diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index 306af1a6d910..1c511647b2fe 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -102,6 +102,8 @@ impl Requester { ) -> Result<()> where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { gum::trace!(target: LOG_TARGET, ?update, "Update fetching heads"); let ActiveLeavesUpdate { activated, deactivated } = update; @@ -128,6 +130,8 @@ impl Requester { ) -> Result<()> where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { let sender = &mut ctx.sender().clone(); let ActivatedLeaf { hash: leaf, .. } = new_head; @@ -186,6 +190,8 @@ impl Requester { ) -> Result<()> where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { for core in cores { match self.fetches.entry(core.candidate_hash) { diff --git a/node/network/availability-distribution/src/requester/session_cache.rs b/node/network/availability-distribution/src/requester/session_cache.rs index 69df02ef80a9..676a90b3ce4a 100644 --- a/node/network/availability-distribution/src/requester/session_cache.rs +++ b/node/network/availability-distribution/src/requester/session_cache.rs @@ -104,6 +104,8 @@ impl SessionCache { ) -> Result> where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, F: FnOnce(&SessionInfo) -> R, { if let Some(o_info) = self.session_info_cache.get(&session_index) { @@ -181,6 +183,8 @@ impl SessionCache { ) -> Result> where Context: overseer::AvailabilityDistributionContextTrait, + ::Sender: + overseer::AvailabilityDistributionSenderTrait, { let info = runtime .get_session_info_by_index(ctx.sender(), relay_parent, session_index) diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 0e6ac9b0077e..55311c5653ae 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -809,6 +809,8 @@ impl Default for State { impl Subsystem for AvailabilityRecoverySubsystem where Context: overseer::AvailabilityRecoveryContextTrait, + ::Sender: + overseer::AvailabilityRecoverySenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -849,6 +851,8 @@ async fn launch_recovery_task( ) -> error::Result<()> where Context: overseer::AvailabilityRecoveryContextTrait, + ::Sender: + overseer::AvailabilityRecoverySenderTrait, { let candidate_hash = receipt.hash(); @@ -901,6 +905,8 @@ async fn handle_recover( ) -> error::Result<()> where Context: overseer::AvailabilityRecoveryContextTrait, + ::Sender: + overseer::AvailabilityRecoverySenderTrait, { let candidate_hash = receipt.hash(); @@ -963,6 +969,8 @@ async fn query_full_data( ) -> error::Result> where Context: overseer::AvailabilityRecoveryContextTrait, + ::Sender: + overseer::AvailabilityRecoverySenderTrait, { let (tx, rx) = oneshot::channel(); ctx.send_message(AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx)) @@ -992,6 +1000,8 @@ impl AvailabilityRecoverySubsystem { async fn run(self, mut ctx: Context) -> SubsystemResult<()> where Context: overseer::AvailabilityRecoveryContextTrait, + ::Sender: + overseer::AvailabilityRecoverySenderTrait, { let mut state = State::default(); let Self { fast_path, mut req_receiver, metrics } = self; diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 3048a9aa8b79..ae607d493188 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -214,6 +214,8 @@ impl BitfieldDistribution { async fn run(self, ctx: Context) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let mut state = ProtocolState::default(); let mut rng = rand::rngs::StdRng::from_entropy(); @@ -318,6 +320,8 @@ impl BitfieldDistribution { async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { gum::trace!(target: LOG_TARGET, ?relay_parent, ?rep, %peer, "reputation change"); @@ -335,6 +339,8 @@ async fn handle_bitfield_distribution( rng: &mut (impl CryptoRng + Rng), ) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let _timer = metrics.time_handle_bitfield_distribution(); @@ -399,6 +405,8 @@ async fn relay_message( rng: &mut (impl CryptoRng + Rng), ) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let relay_parent = message.relay_parent; let span = job_data.span.child("relay-msg"); @@ -486,6 +494,8 @@ async fn process_incoming_peer_message( rng: &mut (impl CryptoRng + Rng), ) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let protocol_v1::BitfieldDistributionMessage::Bitfield(relay_parent, bitfield) = message; gum::trace!( @@ -604,6 +614,8 @@ async fn handle_network_msg( rng: &mut (impl CryptoRng + Rng), ) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let _timer = metrics.time_handle_network_msg(); @@ -684,6 +696,8 @@ async fn handle_peer_view_change( rng: &mut (impl CryptoRng + Rng), ) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let added = state .peer_views @@ -743,6 +757,8 @@ async fn send_tracked_gossip_message( message: BitfieldGossipMessage, ) where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let job_data = if let Some(job_data) = state.per_relay_parent.get_mut(&message.relay_parent) { job_data @@ -775,6 +791,8 @@ async fn send_tracked_gossip_message( impl overseer::Subsystem for BitfieldDistribution where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); @@ -790,6 +808,8 @@ async fn query_basics( ) -> SubsystemResult, SigningContext)>> where Context: overseer::BitfieldDistributionContextTrait, + ::Sender: + overseer::BitfieldDistributionSenderTrait, { let (validators_tx, validators_rx) = oneshot::channel(); let (session_tx, session_rx) = oneshot::channel(); diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index e609577399bf..8e3c4f19c530 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -129,6 +129,7 @@ where Net: Network + Sync, AD: validator_discovery::AuthorityDiscovery + Clone, Context: overseer::NetworkBridgeContextTrait, + ::Sender: overseer::NetworkBridgeSenderTrait, { fn start(mut self, ctx: Context) -> SpawnedSubsystem { // The stream of networking events has to be created at initialization, otherwise the @@ -195,6 +196,7 @@ async fn handle_subsystem_messages( ) -> Result<(), UnexpectedAbort> where Context: overseer::NetworkBridgeContextTrait, + ::Sender: overseer::NetworkBridgeSenderTrait, N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, { @@ -842,6 +844,7 @@ where N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, Context: overseer::NetworkBridgeContextTrait, + ::Sender: overseer::NetworkBridgeSenderTrait, { let shared = Shared::default(); @@ -906,14 +909,18 @@ fn construct_view( View::new(live_heads.take(MAX_VIEW_HEADS), finalized_number) } -fn update_our_view( +fn update_our_view( net: &mut Net, ctx: &mut Context, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, metrics: &Metrics, -) { +) where + Net: Network, + Context: overseer::NetworkBridgeContextTrait, + ::Sender: overseer::NetworkBridgeSenderTrait, +{ let new_view = construct_view(live_heads.iter().map(|v| v.hash), finalized_number); let (validation_peers, collation_peers) = { diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 4cc4c09afa27..86e9e5a8e5a9 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -373,6 +373,8 @@ async fn distribute_collation( ) -> Result<()> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let relay_parent = receipt.descriptor.relay_parent; @@ -499,6 +501,8 @@ async fn determine_our_validators( ) -> Result where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let session_index = runtime.get_session_index_for_child(ctx.sender(), relay_parent).await?; let info = &runtime @@ -529,6 +533,8 @@ where async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let declare_signature_payload = protocol_v1::declare_signature_payload(&state.local_peer_id); @@ -552,6 +558,8 @@ where async fn connect_to_validators(ctx: &mut Context, validator_ids: Vec) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { // ignore address resolution failure // will reissue a new request on new collation @@ -575,6 +583,8 @@ async fn advertise_collation( peer: PeerId, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let should_advertise = state .our_validators_groups @@ -636,6 +646,8 @@ async fn process_msg( ) -> Result<()> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { use CollatorProtocolMessage::*; @@ -749,6 +761,8 @@ async fn handle_incoming_peer_message( ) -> Result<()> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { use protocol_v1::CollatorProtocolMessage::*; @@ -829,6 +843,8 @@ async fn handle_incoming_request( ) -> Result<()> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let _span = state .span_per_relay_parent @@ -904,6 +920,8 @@ async fn handle_peer_view_change( view: View, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let current = state.peer_views.entry(peer_id.clone()).or_default(); @@ -925,6 +943,8 @@ async fn handle_network_msg( ) -> Result<()> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { use NetworkBridgeEvent::*; @@ -1018,6 +1038,8 @@ pub(crate) async fn run( ) -> std::result::Result<(), FatalError> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { use OverseerSignal::*; diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 7f3e1c210b6f..860b9ce7a045 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -101,6 +101,8 @@ impl CollatorProtocolSubsystem { async fn run(self, ctx: Context) -> std::result::Result<(), error::FatalError> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => @@ -114,6 +116,8 @@ impl CollatorProtocolSubsystem { impl overseer::Subsystem for CollatorProtocolSubsystem where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -129,6 +133,8 @@ where async fn modify_reputation(ctx: &mut Context, peer: PeerId, rep: Rep) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { gum::trace!( target: LOG_TARGET, diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index f48398585a6c..d11637b2736b 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -633,6 +633,8 @@ fn collator_peer_id( async fn disconnect_peer(ctx: &mut Context, peer_id: PeerId) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { ctx.send_message(NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation)) .await @@ -646,6 +648,8 @@ async fn fetch_collation( id: CollatorId, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let (tx, rx) = oneshot::channel(); @@ -691,6 +695,8 @@ async fn report_collator( id: CollatorId, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(ctx, peer_id, COST_REPORT_BAD).await; @@ -704,6 +710,8 @@ async fn note_good_collation( id: CollatorId, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(ctx, peer_id, BENEFIT_NOTIFY_GOOD).await; @@ -718,6 +726,8 @@ async fn notify_collation_seconded( statement: SignedFullStatement, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let wire_message = protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement.into()); @@ -759,6 +769,8 @@ async fn request_collation( result: oneshot::Sender<(CandidateReceipt, PoV)>, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { if !state.view.contains(&relay_parent) { gum::debug!( @@ -825,6 +837,8 @@ async fn process_incoming_peer_message( msg: protocol_v1::CollatorProtocolMessage, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { use protocol_v1::CollatorProtocolMessage::*; use sp_runtime::traits::AppVerify; @@ -1013,6 +1027,8 @@ async fn handle_our_view_change( ) -> Result<()> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let old_view = std::mem::replace(&mut state.view, view); @@ -1071,6 +1087,8 @@ async fn handle_network_msg( ) -> Result<()> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { use NetworkBridgeEvent::*; @@ -1108,6 +1126,8 @@ async fn process_msg( state: &mut State, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { use CollatorProtocolMessage::*; @@ -1210,6 +1230,8 @@ pub(crate) async fn run( ) -> std::result::Result<(), crate::error::FatalError> where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { let mut state = State { metrics, ..Default::default() }; @@ -1294,13 +1316,17 @@ async fn poll_requests( } /// Dequeue another collation and fetch. -async fn dequeue_next_collation_and_fetch( +async fn dequeue_next_collation_and_fetch( ctx: &mut Context, state: &mut State, relay_parent: Hash, // The collator we tried to fetch from last. previous_fetch: CollatorId, -) { +) where + Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, +{ if let Some((next, id)) = state .collations_per_relay_parent .get_mut(&relay_parent) @@ -1330,6 +1356,8 @@ async fn handle_collation_fetched_result( (mut collation_event, res): PendingCollationFetch, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { // If no prior collation for this relay parent has been seconded, then // memorize the `collation_event` for that `relay_parent`, such that we may @@ -1395,6 +1423,8 @@ async fn disconnect_inactive_peers( peers: &HashMap, ) where Context: overseer::CollatorProtocolContextTrait, + ::Sender: + overseer::CollatorProtocolSenderTrait, { for (peer, peer_data) in peers { if peer_data.is_inactive(&eviction_policy) { diff --git a/node/network/dispute-distribution/src/lib.rs b/node/network/dispute-distribution/src/lib.rs index 6b710921af68..65ae2984cb31 100644 --- a/node/network/dispute-distribution/src/lib.rs +++ b/node/network/dispute-distribution/src/lib.rs @@ -116,7 +116,9 @@ pub struct DisputeDistributionSubsystem { impl overseer::Subsystem for DisputeDistributionSubsystem where - Context: overseer::DisputeDistributionContextTrait + Sync + Send, + Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait + Sync + Send, AD: AuthorityDiscovery + Clone, { fn start(self, ctx: Context) -> SpawnedSubsystem { @@ -159,7 +161,9 @@ where /// Start processing work as passed on from the Overseer. async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> where - Context: overseer::DisputeDistributionContextTrait + Sync + Send, + Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait + Sync + Send, { let receiver = DisputesReceiver::new( ctx.sender().clone(), @@ -206,6 +210,8 @@ where ) -> Result where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { match signal { OverseerSignal::Conclude => return Ok(SignalResult::Conclude), @@ -225,6 +231,8 @@ where ) -> Result<()> where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { match msg { DisputeDistributionMessage::SendDispute(dispute_msg) => @@ -250,6 +258,8 @@ impl MuxedMessage { ) -> Self where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { // We are only fusing here to make `select` happy, in reality we will quit if the stream // ends. diff --git a/node/network/dispute-distribution/src/sender/mod.rs b/node/network/dispute-distribution/src/sender/mod.rs index 74816cacc31d..80684c9bd0b7 100644 --- a/node/network/dispute-distribution/src/sender/mod.rs +++ b/node/network/dispute-distribution/src/sender/mod.rs @@ -87,6 +87,8 @@ impl DisputeSender { ) -> Result<()> where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { let req: DisputeRequest = msg.into(); let candidate_hash = req.0.candidate_receipt.hash(); @@ -125,6 +127,8 @@ impl DisputeSender { ) -> Result<()> where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { let ActiveLeavesUpdate { activated, deactivated } = update; let deactivated: HashSet<_> = deactivated.into_iter().collect(); @@ -193,6 +197,8 @@ impl DisputeSender { ) -> Result<()> where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { let (session_index, candidate_hash) = dispute; // A relay chain head is required as context for receiving session info information from runtime and @@ -319,6 +325,8 @@ impl DisputeSender { ) -> Result where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { let new_sessions = get_active_session_indices(ctx, runtime, &self.active_heads).await?; let new_sessions_raw: HashSet<_> = new_sessions.keys().collect(); @@ -340,6 +348,8 @@ async fn get_active_session_indices( ) -> Result> where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { let mut indeces = HashMap::new(); // Iterate all heads we track as active and fetch the child' session indices. @@ -356,6 +366,8 @@ async fn get_active_disputes( ) -> JfyiErrorResult> where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { let (tx, rx) = oneshot::channel(); @@ -372,6 +384,8 @@ async fn get_candidate_votes( ) -> JfyiErrorResult> where Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, { let (tx, rx) = oneshot::channel(); // Caller scope is in `update_leaves` and this is bounded by fork count. diff --git a/node/network/dispute-distribution/src/sender/send_task.rs b/node/network/dispute-distribution/src/sender/send_task.rs index 8f0fe5453629..679765331da1 100644 --- a/node/network/dispute-distribution/src/sender/send_task.rs +++ b/node/network/dispute-distribution/src/sender/send_task.rs @@ -99,14 +99,19 @@ impl TaskResult { impl SendTask { /// Initiates sending a dispute message to peers. - pub async fn new( + pub async fn new( ctx: &mut Context, runtime: &mut RuntimeInfo, active_sessions: &HashMap, tx: mpsc::Sender, request: DisputeRequest, metrics: &Metrics, - ) -> Result { + ) -> Result + where + Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, + { let mut send_task = Self { request, deliveries: HashMap::new(), has_failed_sends: false, tx }; send_task.refresh_sends(ctx, runtime, active_sessions, metrics).await?; @@ -117,13 +122,18 @@ impl SendTask { /// /// This function is called at construction and should also be called whenever a session change /// happens and on a regular basis to ensure we are retrying failed attempts. - pub async fn refresh_sends( + pub async fn refresh_sends( &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, active_sessions: &HashMap, metrics: &Metrics, - ) -> Result<()> { + ) -> Result<()> + where + Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, + { let new_authorities = self.get_relevant_validators(ctx, runtime, active_sessions).await?; let add_authorities = new_authorities @@ -194,12 +204,18 @@ impl SendTask { /// /// This is all parachain validators of the session the candidate occurred and all authorities /// of all currently active sessions, determined by currently active heads. - async fn get_relevant_validators( + + async fn get_relevant_validators( &self, ctx: &mut Context, runtime: &mut RuntimeInfo, active_sessions: &HashMap, - ) -> Result> { + ) -> Result> + where + Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, + { let ref_head = self.request.0.candidate_receipt.descriptor.relay_parent; // Retrieve all authorities which participated in the parachain consensus of the session // in which the candidate was backed. @@ -238,13 +254,18 @@ impl SendTask { /// Start sending of the given message to all given authorities. /// /// And spawn tasks for handling the response. -async fn send_requests( +async fn send_requests( ctx: &mut Context, tx: mpsc::Sender, receivers: Vec, req: DisputeRequest, metrics: &Metrics, -) -> Result> { +) -> Result> +where + Context: overseer::DisputeDistributionContextTrait, + ::Sender: + overseer::DisputeDistributionSenderTrait, +{ let mut statuses = HashMap::with_capacity(receivers.len()); let mut reqs = Vec::with_capacity(receivers.len()); diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 7f0244615d1b..9a882f07ab34 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -141,6 +141,8 @@ where async fn run(mut self, mut ctx: Context) -> Self where Context: overseer::GossipSupportContextTrait, + ::Sender: + overseer::GossipSupportSenderTrait, { fn get_connectivity_check_delay() -> Delay { Delay::new(LOW_CONNECTIVITY_WARN_DELAY) @@ -587,6 +589,7 @@ fn matrix_neighbors( impl overseer::Subsystem for GossipSupport where Context: overseer::GossipSupportContextTrait, + ::Sender: overseer::GossipSupportSenderTrait, AD: AuthorityDiscovery + Clone, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 0ff27cdde525..8338a0a2cbc2 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -131,6 +131,8 @@ impl overseer::Subsystem where Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision @@ -630,11 +632,16 @@ enum MuxedMessage { } impl MuxedMessage { - async fn receive( + async fn receive( ctx: &mut Context, from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, - ) -> MuxedMessage { + ) -> MuxedMessage + where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, + { // We are only fusing here to make `select` happy, in reality we will quit if one of those // streams end: let from_overseer = ctx.recv().fuse(); @@ -888,9 +895,7 @@ fn check_statement_signature( /// circulates the statement to all peers who have not seen it yet, and /// sends all statements dependent on that statement to peers who could previously not receive /// them but now can. -async fn circulate_statement_and_dependents< - Context: overseer::StatementDistributionContextTrait, ->( +async fn circulate_statement_and_dependents( gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &mut HashMap, @@ -900,7 +905,11 @@ async fn circulate_statement_and_dependents< priority_peers: Vec, metrics: &Metrics, rng: &mut impl rand::Rng, -) { +) where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, +{ let active_head = match active_heads.get_mut(&relay_parent) { Some(res) => res, None => return, @@ -1006,7 +1015,7 @@ fn is_statement_large(statement: &SignedFullStatement) -> (bool, Option) /// Circulates a statement to all peers who have not seen it yet, and returns /// an iterator over peers who need to have dependent statements sent. -async fn circulate_statement<'a, Context: overseer::StatementDistributionContextTrait>( +async fn circulate_statement<'a, Context>( gossip_peers: &HashSet, peers: &mut HashMap, ctx: &mut Context, @@ -1015,7 +1024,12 @@ async fn circulate_statement<'a, Context: overseer::StatementDistributionContext mut priority_peers: Vec, metrics: &Metrics, rng: &mut impl rand::Rng, -) -> Vec { +) -> Vec +where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, +{ let fingerprint = stored.fingerprint(); let mut peers_to_send: Vec = peers @@ -1095,7 +1109,7 @@ async fn circulate_statement<'a, Context: overseer::StatementDistributionContext } /// Send all statements about a given candidate hash to a peer. -async fn send_statements_about( +async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, ctx: &mut Context, @@ -1103,7 +1117,11 @@ async fn send_statements_about::Sender: + overseer::StatementDistributionSenderTrait, +{ for statement in active_head.statements_about(candidate_hash) { let fingerprint = statement.fingerprint(); if !peer_data.can_send(&relay_parent, &fingerprint) { @@ -1128,14 +1146,18 @@ async fn send_statements_about( +async fn send_statements( peer: PeerId, peer_data: &mut PeerData, ctx: &mut Context, relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, -) { +) where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, +{ for statement in active_head.statements() { let fingerprint = statement.fingerprint(); if !peer_data.can_send(&relay_parent, &fingerprint) { @@ -1158,11 +1180,12 @@ async fn send_statements( } } -async fn report_peer( - ctx: &mut Context, - peer: PeerId, - rep: Rep, -) { +async fn report_peer(ctx: &mut Context, peer: PeerId, rep: Rep) +where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, +{ ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await } @@ -1173,17 +1196,19 @@ async fn report_peer( /// your statement. /// /// If the message was large, but the result has been fetched already that one is returned. -async fn retrieve_statement_from_message< - 'a, - Context: overseer::StatementDistributionContextTrait, ->( +async fn retrieve_statement_from_message<'a, Context>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, active_head: &'a mut ActiveHeadData, ctx: &mut Context, req_sender: &mpsc::Sender, metrics: &Metrics, -) -> Option { +) -> Option +where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, +{ let fingerprint = message.get_fingerprint(); let candidate_hash = *fingerprint.0.candidate_hash(); @@ -1268,13 +1293,18 @@ async fn retrieve_statement_from_message< /// Launch request for a large statement and get tracking status. /// /// Returns `None` if spawning task failed. -async fn launch_request( +async fn launch_request( meta: StatementMetadata, peer: PeerId, req_sender: mpsc::Sender, ctx: &mut Context, metrics: &Metrics, -) -> Option { +) -> Option +where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, +{ let (task, handle) = fetch(meta.relay_parent, meta.candidate_hash, vec![peer], req_sender, metrics.clone()) .remote_handle(); @@ -1299,10 +1329,7 @@ async fn launch_request( /// Handle incoming message and circulate it to peers, if we did not know it already. /// -async fn handle_incoming_message_and_circulate< - 'a, - Context: overseer::StatementDistributionContextTrait, ->( +async fn handle_incoming_message_and_circulate<'a, Context, R>( peer: PeerId, gossip_peers: &HashSet, peers: &mut HashMap, @@ -1312,8 +1339,13 @@ async fn handle_incoming_message_and_circulate< message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, - rng: &mut impl rand::Rng, -) { + rng: &mut R, +) where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, + R: rand::Rng, +{ let handled_incoming = match peers.get_mut(&peer) { Some(data) => handle_incoming_message( @@ -1359,7 +1391,7 @@ async fn handle_incoming_message_and_circulate< // // This function checks the signature and ensures the statement is compatible with our // view. It also notifies candidate backing if the statement was previously unknown. -async fn handle_incoming_message<'a, Context: overseer::StatementDistributionContextTrait>( +async fn handle_incoming_message<'a, Context>( peer: PeerId, peer_data: &mut PeerData, active_heads: &'a mut HashMap, @@ -1368,7 +1400,12 @@ async fn handle_incoming_message<'a, Context: overseer::StatementDistributionCon message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, -) -> Option<(Hash, StoredStatement<'a>)> { +) -> Option<(Hash, StoredStatement<'a>)> +where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, +{ let relay_parent = message.get_relay_parent(); let _ = metrics.time_network_bridge_update_v1("handle_incoming_message"); @@ -1556,9 +1593,7 @@ async fn handle_incoming_message<'a, Context: overseer::StatementDistributionCon } /// Update a peer's view. Sends all newly unlocked statements based on the previous -async fn update_peer_view_and_maybe_send_unlocked< - Context: overseer::StatementDistributionContextTrait, ->( +async fn update_peer_view_and_maybe_send_unlocked( peer: PeerId, gossip_peers: &HashSet, peer_data: &mut PeerData, @@ -1566,8 +1601,13 @@ async fn update_peer_view_and_maybe_send_unlocked< active_heads: &HashMap, new_view: View, metrics: &Metrics, - rng: &mut impl rand::Rng, -) { + rng: &mut R, +) where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, + R: rand::Rng, +{ let old_view = std::mem::replace(&mut peer_data.view, new_view); // Remove entries for all relay-parents in the old view but not the new. @@ -1597,7 +1637,7 @@ async fn update_peer_view_and_maybe_send_unlocked< } } -async fn handle_network_update( +async fn handle_network_update( peers: &mut HashMap, gossip_peers: &mut HashSet, authorities: &mut HashMap, @@ -1607,8 +1647,13 @@ async fn handle_network_update, update: NetworkBridgeEvent, metrics: &Metrics, - rng: &mut impl rand::Rng, -) { + rng: &mut R, +) where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, + R: rand::Rng, +{ match update { NetworkBridgeEvent::PeerConnected(peer, role, _, maybe_authority) => { gum::trace!(target: LOG_TARGET, ?peer, ?role, "Peer connected"); @@ -1713,10 +1758,12 @@ impl StatementDistributionSubsystem { Self { keystore, req_receiver: Some(req_receiver), metrics, rng } } - async fn run( - mut self, - mut ctx: Context, - ) -> std::result::Result<(), FatalError> { + async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> + where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, + { let mut peers: HashMap = HashMap::new(); let mut gossip_peers: HashSet = HashSet::new(); let mut authorities: HashMap = HashMap::new(); @@ -1832,7 +1879,7 @@ impl StatementDistributionSubsystem { Ok(()) } - async fn handle_requester_message( + async fn handle_requester_message( &mut self, ctx: &mut Context, gossip_peers: &HashSet, @@ -1841,7 +1888,12 @@ impl StatementDistributionSubsystem { recent_outdated_heads: &RecentOutdatedHeads, req_sender: &mpsc::Sender, message: RequesterMessage, - ) -> JfyiErrorResult<()> { + ) -> JfyiErrorResult<()> + where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, + { match message { RequesterMessage::Finished { relay_parent, @@ -1940,7 +1992,7 @@ impl StatementDistributionSubsystem { Ok(()) } - async fn handle_subsystem_message( + async fn handle_subsystem_message( &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, @@ -1951,7 +2003,12 @@ impl StatementDistributionSubsystem { recent_outdated_heads: &mut RecentOutdatedHeads, req_sender: &mpsc::Sender, message: FromOverseer, - ) -> Result { + ) -> Result + where + Context: overseer::StatementDistributionContextTrait, + ::Sender: + overseer::StatementDistributionSenderTrait, + { let metrics = &self.metrics; match message { diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index bccc81f5ed97..c2b54160cc86 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -17,10 +17,7 @@ //! A utility for fetching all unknown blocks based on a new chain-head hash. use futures::{channel::oneshot, prelude::*}; -use polkadot_node_subsystem::{ - messages::ChainApiMessage, - SubsystemSender, -}; +use polkadot_node_subsystem::{messages::ChainApiMessage, SubsystemSender}; use polkadot_primitives::v2::{BlockNumber, Hash, Header}; /// Given a new chain-head hash, this determines the hashes of all new blocks we should track diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 46318c20c771..0648ac9bcdb3 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -26,9 +26,7 @@ use polkadot_node_subsystem::{ errors::{RuntimeApiError, SubsystemError}, - messages::{ - BoundToRelayParent, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, - }, + messages::{BoundToRelayParent, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender}, overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemSender, }; diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 0028116901d4..fc660a9dc6df 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -25,9 +25,7 @@ use sp_application_crypto::AppKey; use sp_core::crypto::ByteArray; use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; -use polkadot_node_subsystem::{ - messages::RuntimeApiMessage, overseer, SubsystemSender, -}; +use polkadot_node_subsystem::{messages::RuntimeApiMessage, overseer, SubsystemSender}; use polkadot_primitives::v2::{ CandidateEvent, CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, ScrapedOnChainVotes, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, From bcb08b8400e20c544130f66911d2e7b90b231f13 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 08:16:49 +0200 Subject: [PATCH 065/139] chore fixes --- node/core/pvf-checker/src/lib.rs | 10 ++-------- node/malus/src/interceptor.rs | 30 +++++++++++++----------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index 6951d485e909..5b4bf2f00c0e 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -129,14 +129,8 @@ async fn run( metrics: Metrics, ) -> SubsystemResult<()> where - Context: overseer::SubsystemContext< - Message = PvfCheckerMessage, - OutgoingMessages = overseer::PvfCheckerOutgoingMessages, - Signal = OverseerSignal, - Error = SubsystemError, - // Sender is ommitted here - >, - ::Sender: overseer::PvfCheckerSenderTrait, + Context: overseer::PvfCheckerContextTrait, + ::Sender: overseer::PvfCheckerSenderTrait, { let mut state = State { credentials: None, diff --git a/node/malus/src/interceptor.rs b/node/malus/src/interceptor.rs index 5da91fd42b66..234d5afacf24 100644 --- a/node/malus/src/interceptor.rs +++ b/node/malus/src/interceptor.rs @@ -21,7 +21,7 @@ //! messages on the overseer level. use polkadot_node_subsystem::*; -pub use polkadot_node_subsystem::{messages::AllMessages, overseer, FromOverseer}; +pub use polkadot_node_subsystem::{overseer, FromOverseer}; use std::{future::Future, pin::Pin}; /// Filter incoming and outgoing messages. @@ -30,7 +30,7 @@ where Sender: overseer::SubsystemSender + Clone + 'static, { /// The message type the original subsystem handles incoming. - type Message: Send + 'static; + type Message: AssociateOutgoing + Send + 'static; /// Filter messages that are to be received by /// the subsystem. @@ -46,7 +46,7 @@ where } /// Modify outgoing messages. - fn intercept_outgoing(&self, msg: AllMessages) -> Option { + fn intercept_outgoing(&self, msg: AssociateOutgoing::::OutgoingMessages) -> Option { Some(msg) } } @@ -59,13 +59,12 @@ pub struct InterceptedSender { } #[async_trait::async_trait] -impl overseer::SubsystemSender for InterceptedSender +impl overseer::SubsystemSender for InterceptedSender where - Sender: overseer::SubsystemSender - + overseer::SubsystemSender<>::Message>, + Sender: overseer::SubsystemSender<>::Message>, Fil: MessageInterceptor, { - async fn send_message(&mut self, msg: AllMessages) { + async fn send_message(&mut self, msg: M) { if let Some(msg) = self.message_filter.intercept_outgoing(msg) { self.inner.send_message(msg).await; } @@ -73,7 +72,7 @@ where async fn send_messages(&mut self, msgs: T) where - T: IntoIterator + Send, + T: IntoIterator + Send, T::IntoIter: Send, { for msg in msgs { @@ -81,7 +80,7 @@ where } } - fn send_unbounded_message(&mut self, msg: AllMessages) { + fn send_unbounded_message(&mut self, msg: M) { if let Some(msg) = self.message_filter.intercept_outgoing(msg) { self.inner.send_unbounded_message(msg); } @@ -91,7 +90,7 @@ where /// A subsystem context, that filters the outgoing messages. pub struct InterceptedContext where - Context: overseer::SubsystemContext + SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor<::Sender>, ::Sender: overseer::SubsystemSender< ::Sender>>::Message, @@ -104,7 +103,7 @@ where impl InterceptedContext where - Context: overseer::SubsystemContext + SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor< ::Sender, Message = ::Message, @@ -125,13 +124,11 @@ where #[async_trait::async_trait] impl overseer::SubsystemContext for InterceptedContext where - Context: overseer::SubsystemContext + SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor< ::Sender, Message = ::Message, >, - ::AllMessages: - From<::Message>, ::Sender: overseer::SubsystemSender< ::Sender>>::Message, >, @@ -139,7 +136,6 @@ where type Message = ::Message; type Sender = InterceptedSender<::Sender, Fil>; type Error = ::Error; - type AllMessages = ::AllMessages; type Signal = ::Signal; async fn try_recv(&mut self) -> Result>, ()> { @@ -200,9 +196,9 @@ impl InterceptedSubsystem { impl overseer::Subsystem for InterceptedSubsystem where - Context: overseer::SubsystemContext + SubsystemContext + Sync + Send, + Context: overseer::SubsystemContext + Sync + Send, Sub: overseer::Subsystem, SubsystemError>, - InterceptedContext: overseer::SubsystemContext + SubsystemContext, + InterceptedContext: overseer::SubsystemContext, Interceptor: MessageInterceptor< ::Sender, Message = ::Message, From 41b59eec1244013776a31df992422b72a8d4f734 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 08:17:07 +0200 Subject: [PATCH 066/139] overseer-gen: split tests and misc, for readability --- node/overseer/overseer-gen/examples/dummy.rs | 165 ------------------- node/overseer/overseer-gen/examples/duo.rs | 78 +++++++++ node/overseer/overseer-gen/examples/misc.rs | 91 ++++++++++ node/overseer/overseer-gen/examples/solo.rs | 43 +++++ 4 files changed, 212 insertions(+), 165 deletions(-) delete mode 100644 node/overseer/overseer-gen/examples/dummy.rs create mode 100644 node/overseer/overseer-gen/examples/duo.rs create mode 100644 node/overseer/overseer-gen/examples/misc.rs create mode 100644 node/overseer/overseer-gen/examples/solo.rs diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs deleted file mode 100644 index aedb3feda676..000000000000 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! A dummy to be used with cargo expand - -use polkadot_node_network_protocol::WrongVariant; -use polkadot_overseer_gen::{SpawnNamed, *}; -use std::collections::HashMap; - -/// Concrete subsystem implementation for `MsgStrukt` msg type. -#[derive(Default)] -pub struct AwesomeSubSys; - -impl ::polkadot_overseer_gen::Subsystem for AwesomeSubSys -where - Ctx: AwesomeSubSysContextTrait, -{ - fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { - let mut sender = ctx.sender().clone(); - ctx.spawn( - "AwesomeSubsys", - Box::pin(async move { - sender.send_message(Plinko).await; - }), - ) - .unwrap(); - unimplemented!("starting yay!") - } -} - -#[derive(Default)] -pub struct GoblinTower; - -impl ::polkadot_overseer_gen::Subsystem for GoblinTower -where - Ctx: GoblinTowerContextTrait, -{ - fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { - let mut sender = ctx.sender().clone(); - ctx.spawn( - "GoblinTower", - Box::pin(async move { - sender.send_message(MsgStrukt(8u8)).await; - }), - ) - .unwrap(); - unimplemented!("welcum") - } -} - -/// A signal sent by the overseer. -#[derive(Debug, Clone)] -pub struct SigSigSig; - -/// The external event. -#[derive(Debug, Clone)] -pub struct EvX; - -impl EvX { - pub fn focus<'a, T>(&'a self) -> Result { - unimplemented!("dispatch") - } -} - -#[derive(Debug, Clone, Copy)] -pub struct Yikes; - -impl std::fmt::Display for Yikes { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "yikes!") - } -} - -impl std::error::Error for Yikes {} - -impl From for Yikes { - fn from(_: polkadot_overseer_gen::OverseerError) -> Yikes { - Yikes - } -} - -impl From for Yikes { - fn from(_: polkadot_overseer_gen::mpsc::SendError) -> Yikes { - Yikes - } -} - -#[derive(Debug, Clone)] -pub struct MsgStrukt(u8); - -#[derive(Debug, Clone, Copy)] -pub struct Plinko; - -impl From for MsgStrukt { - fn from(_event: NetworkMsg) -> Self { - MsgStrukt(1u8) - } -} - -#[derive(Debug, Clone, Copy)] -pub enum NetworkMsg { - A, - B, - C, -} - -impl NetworkMsg { - fn focus(&self) -> Result { - Ok(match self { - Self::B => return Err(WrongVariant), - Self::A | Self::C => self.clone(), - }) - } -} - -#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] -struct Xxx { - #[subsystem(consumes: MsgStrukt, sends: [Plinko])] - sub0: AwesomeSubSys, - - #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: [MsgStrukt])] - plinkos: GoblinTower, - - i_like_pi: f64, - i_like_generic: T, - i_like_hash: HashMap, -} - -#[derive(Debug, Clone)] -struct DummySpawner; - -impl SpawnNamed for DummySpawner { - fn spawn_blocking( - &self, - task_name: &'static str, - subsystem_name: Option<&'static str>, - _future: futures::future::BoxFuture<'static, ()>, - ) { - unimplemented!("spawn blocking {} {}", task_name, subsystem_name.unwrap_or("default")) - } - - fn spawn( - &self, - task_name: &'static str, - subsystem_name: Option<&'static str>, - _future: futures::future::BoxFuture<'static, ()>, - ) { - unimplemented!("spawn {} {}", task_name, subsystem_name.unwrap_or("default")) - } -} - -#[derive(Debug, Clone)] -struct DummyCtx; - -fn main() { - let (overseer, _handle): (Xxx<_, f64>, _) = Xxx::builder() - .sub0(AwesomeSubSys::default()) - .plinkos(GoblinTower::default()) - .i_like_pi(::std::f64::consts::PI) - .i_like_generic(42.0) - .i_like_hash(HashMap::new()) - .spawner(DummySpawner) - .build() - .unwrap(); - assert_eq!(overseer.i_like_pi.floor() as i8, 3); - assert_eq!(overseer.i_like_generic.floor() as i8, 42); - assert_eq!(overseer.i_like_hash.len() as i8, 0); -} diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs new file mode 100644 index 000000000000..962a25c22c62 --- /dev/null +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -0,0 +1,78 @@ +//! A dummy to be used with cargo expand + +use polkadot_node_network_protocol::WrongVariant; +use polkadot_overseer_gen::{SpawnNamed, *}; +use std::collections::HashMap; +mod misc; + +pub use self::misc::*; + +/// Concrete subsystem implementation for `MsgStrukt` msg type. +#[derive(Default)] +pub struct AwesomeSubSys; + +impl overseer::Subsystem for AwesomeSubSys +where + Ctx: AwesomeSubSysContextTrait, +{ + fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { + let mut sender = ctx.sender().clone(); + ctx.spawn( + "AwesomeSubsys", + Box::pin(async move { + sender.send_message(Plinko).await; + }), + ) + .unwrap(); + unimplemented!("starting yay!") + } +} + +#[derive(Default)] +pub struct GoblinTower; + +impl overseer::Subsystem for GoblinTower +where + Ctx: GoblinTowerContextTrait, + GoblinTowerContextTrait, +{ + fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { + let mut sender = ctx.sender().clone(); + ctx.spawn( + "GoblinTower", + Box::pin(async move { + sender.send_message(MsgStrukt(8u8)).await; + }), + ) + .unwrap(); + unimplemented!("welcum") + } +} + +#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] +struct Duo { + #[subsystem(consumes: MsgStrukt, sends: [Plinko])] + sub0: AwesomeSubSys, + + #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: [MsgStrukt])] + plinkos: GoblinTower, + + i_like_pi: f64, + i_like_generic: T, + i_like_hash: HashMap, +} + +fn main() { + let (overseer, _handle): (Duo<_, f64>, _) = Duo::builder() + .sub0(AwesomeSubSys::default()) + .plinkos(GoblinTower::default()) + .i_like_pi(::std::f64::consts::PI) + .i_like_generic(42.0) + .i_like_hash(HashMap::new()) + .spawner(DummySpawner) + .build() + .unwrap(); + assert_eq!(overseer.i_like_pi.floor() as i8, 3); + assert_eq!(overseer.i_like_generic.floor() as i8, 42); + assert_eq!(overseer.i_like_hash.len() as i8, 0); +} diff --git a/node/overseer/overseer-gen/examples/misc.rs b/node/overseer/overseer-gen/examples/misc.rs new file mode 100644 index 000000000000..85b50b9a482b --- /dev/null +++ b/node/overseer/overseer-gen/examples/misc.rs @@ -0,0 +1,91 @@ +use polkadot_overseer_gen::{SpawnNamed, *}; + +#[derive(Debug, Clone)] +pub struct DummySpawner; + +impl SpawnNamed for DummySpawner { + fn spawn_blocking( + &self, + task_name: &'static str, + subsystem_name: Option<&'static str>, + _future: futures::future::BoxFuture<'static, ()>, + ) { + unimplemented!("spawn blocking {} {}", task_name, subsystem_name.unwrap_or("default")) + } + + fn spawn( + &self, + task_name: &'static str, + subsystem_name: Option<&'static str>, + _future: futures::future::BoxFuture<'static, ()>, + ) { + unimplemented!("spawn {} {}", task_name, subsystem_name.unwrap_or("default")) + } +} + + + +/// A signal sent by the overseer. +#[derive(Debug, Clone)] +pub struct SigSigSig; + +/// The external event. +#[derive(Debug, Clone)] +pub struct EvX; + +impl EvX { + pub fn focus<'a, T>(&'a self) -> Result { + unimplemented!("dispatch") + } +} + +#[derive(Debug, Clone, Copy)] +pub struct Yikes; + +impl std::fmt::Display for Yikes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "yikes!") + } +} + +impl std::error::Error for Yikes {} + +impl From for Yikes { + fn from(_: polkadot_overseer_gen::OverseerError) -> Yikes { + Yikes + } +} + +impl From for Yikes { + fn from(_: polkadot_overseer_gen::mpsc::SendError) -> Yikes { + Yikes + } +} + +#[derive(Debug, Clone)] +pub struct MsgStrukt(pub u8); + +#[derive(Debug, Clone, Copy)] +pub struct Plinko; + +impl From for MsgStrukt { + fn from(_event: NetworkMsg) -> Self { + MsgStrukt(1u8) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum NetworkMsg { + A, + B, + C, +} + +impl NetworkMsg { + pub fn focus(&self) -> Result { + Ok(match self { + Self::B => return Err(WrongVariant), + Self::A | Self::C => self.clone(), + }) + } +} diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs new file mode 100644 index 000000000000..eb27d9b53eaa --- /dev/null +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -0,0 +1,43 @@ +//! A minimal demo to be used with cargo expand. + +use polkadot_overseer_gen::{SpawnNamed, self as overseer, *}; +mod misc; + +pub use self::misc::*; + +#[derive(Default)] +pub struct GoblinTower; + +impl overseer::Subsystem for GoblinTower +where + Ctx: GoblinTowerContextTrait, + ::Sender: GoblinTowerSenderTrait, + ::Sender: GoblinTowerSenderTrait, +{ + fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { + let mut sender = ctx.sender().clone(); + ctx.spawn( + "GoblinTower", + Box::pin(async move { + sender.send_message(MsgStrukt(8u8)).await; + }), + ) + .unwrap(); + unimplemented!("welcum") + } +} + + +#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] +struct Solo { + #[subsystem(no_dispatch, consumes: Plinko, sends: [MsgStrukt])] + goblin_tower: GoblinTower, +} + +fn main() { + let (overseer, _handle): (Solo<_, f64>, _) = Solo::builder() + .goblin_tower(GoblinTower::default()) + .spawner(DummySpawner) + .build() + .unwrap(); +} From 7e39988e1abec6ad05f36fa6206638a48928a293 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 08:17:32 +0200 Subject: [PATCH 067/139] make task name a true literal --- .../overseer-gen/proc-macro/src/impl_builder.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 93d45bfb9b76..371bd7aa4de2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -490,6 +490,11 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { } }); + // Create the string literals for spawn. + let subsystem_name_str_literal = subsystem_name.iter() + .map(|ident| proc_macro2::Literal::string(ident.to_string().replace("_", "-").as_str())) + .collect::>(); + ts.extend(quote! { /// Type used to represent a builder where all fields are initialized and the overseer could be constructed. pub type #initialized_builder<#initialized_builder_generics> = #builder, #( Init<#field_type>, )*>; @@ -577,17 +582,12 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { self.signal_capacity.unwrap_or(SIGNAL_CHANNEL_CAPACITY) ); - // Generate subsystem name based on overseer field name. - let subsystem_string = String::from(stringify!(#subsystem_name)); - // Convert owned `snake case` string to a `kebab case` static str. - let subsystem_static_str = Box::leak(subsystem_string.replace("_", "-").into_boxed_str()); - let ctx = #subsystem_ctx_name::< #consumes >::new( signal_rx, message_rx, channels_out.clone(), to_overseer_tx.clone(), - subsystem_static_str + #subsystem_name_str_literal ); let #subsystem_name: OverseenSubsystem< #consumes > = @@ -598,7 +598,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { unbounded_meter, ctx, #subsystem_name, - subsystem_static_str, + #subsystem_name_str_literal, &mut running_subsystems, )?; )* From 0c6ce5114c81b8f00c7ae050df6c4de53f34ee51 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 08:18:01 +0200 Subject: [PATCH 068/139] make rustc hang --- .../proc-macro/src/impl_subsystem.rs | 296 +++++------------- 1 file changed, 85 insertions(+), 211 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 1ea2d505bf6d..4c7540ae6976 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -169,6 +169,7 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { ts.extend(impl_per_subsystem_helper_traits( info, + subsystem_ctx_name, subsystem_ctx_trait, subsystem_sender_name, subsystem_sender_trait, @@ -361,6 +362,7 @@ pub(crate) fn impl_associate_outgoing_messages( pub(crate) fn impl_per_subsystem_helper_traits( info: &OverseerInfo, + subsystem_ctx_name: &Ident, subsystem_ctx_trait: &Ident, subsystem_sender_name: &Ident, subsystem_sender_trait: &Ident, @@ -383,6 +385,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( + #support_crate ::SubsystemSender< #outgoing > )* + Send + + 'static }; ts.extend(quote! { @@ -401,8 +404,10 @@ pub(crate) fn impl_per_subsystem_helper_traits( #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, #all_messages_wrapper: From< #outgoing_wrapper >, #all_messages_wrapper: From< #consumes >, - ::Sender: #subsystem_sender_trait, - ::Sender: #acc_sender_trait_bounds, + #support_crate :: #outgoing_wrapper: #( From< #outgoing > )+*, + ::Sender: + #subsystem_sender_trait + + #acc_sender_trait_bounds, }; ts.extend(quote! { @@ -432,232 +437,31 @@ pub(crate) fn impl_per_subsystem_helper_traits( >, #where_clause { - type Sender = ::Sender; + type Sender = ::Sender; } }); - // // impl the subsystem context trait (alt!) - // ts.extend(quote!{ - - // #[#support_crate ::async_trait] - // impl #support_crate ::SubsystemContext for #subsystem_ctx_name <#consumes> - // where - // #consumes: AssociateOutgoing, - // // : <#consumes as AssociateOutgoing>::OutgoingMessages, - // #all_messages_wrapper: From< #consumes >, - // #subsystem_sender_name: - // #support_crate ::SubsystemSender< #outgoing_wrapper > - // #( - // + #support_crate ::SubsystemSender< #outgoing > - // )* - // , - // { - // type Message = #consumes; - // type Signal = #signal; - // type OutgoingMessages = #outgoing_wrapper; - // // type AllMessages = #all_messages_wrapper; - // type Sender = #subsystem_sender_name; - // type Error = #error_ty; - - // async fn try_recv(&mut self) -> ::std::result::Result>, ()> { - // match #support_crate ::poll!(self.recv()) { - // #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), - // #support_crate ::Poll::Pending => Ok(None), - // } - // } - - // async fn recv(&mut self) -> ::std::result::Result, Self::Error> { - // loop { - // // If we have a message pending an overseer signal, we only poll for signals - // // in the meantime. - // if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { - // if needs_signals_received <= self.signals_received.load() { - // return Ok( #support_crate ::FromOverseer::Communication { msg }); - // } else { - // self.pending_incoming = Some((needs_signals_received, msg)); - - // // wait for next signal. - // let signal = self.signals.next().await - // .ok_or(#support_crate ::OverseerError::Context( - // "Signal channel is terminated and empty." - // .to_owned() - // ))?; - - // self.signals_received.inc(); - // return Ok( #support_crate ::FromOverseer::Signal(signal)) - // } - // } - - // let mut await_message = self.messages.next().fuse(); - // let mut await_signal = self.signals.next().fuse(); - // let signals_received = self.signals_received.load(); - // let pending_incoming = &mut self.pending_incoming; - - // // Otherwise, wait for the next signal or incoming message. - // let from_overseer = #support_crate ::futures::select_biased! { - // signal = await_signal => { - // let signal = signal - // .ok_or( #support_crate ::OverseerError::Context( - // "Signal channel is terminated and empty." - // .to_owned() - // ))?; - - // #support_crate ::FromOverseer::Signal(signal) - // } - // msg = await_message => { - // let packet = msg - // .ok_or( #support_crate ::OverseerError::Context( - // "Message channel is terminated and empty." - // .to_owned() - // ))?; - - // if packet.signals_received > signals_received { - // // wait until we've received enough signals to return this message. - // *pending_incoming = Some((packet.signals_received, packet.message)); - // continue; - // } else { - // // we know enough to return this message. - // #support_crate ::FromOverseer::Communication { msg: packet.message} - // } - // } - // }; - - // if let #support_crate ::FromOverseer::Signal(_) = from_overseer { - // self.signals_received.inc(); - // } - - // return Ok(from_overseer); - // } - // } - - // fn sender(&mut self) -> &mut Self::Sender { - // &mut self.to_subsystems - // } - - // fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - // -> ::std::result::Result<(), #error_ty> - // { - // self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { - // name, - // subsystem: Some(self.name()), - // s, - // }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; - // Ok(()) - // } - - // fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - // -> ::std::result::Result<(), #error_ty> - // { - // self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { - // name, - // subsystem: Some(self.name()), - // s, - // }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; - // Ok(()) - // } - // } - // }); - ts -} - -/// Implement a builder pattern for the `Overseer`-type, -/// which acts as the gateway to constructing the overseer. -pub(crate) fn impl_subsystem_context( - info: &OverseerInfo, - subsystem_sender_name: &Ident, - subsystem_ctx_name: &Ident, -) -> TokenStream { - let all_messages_wrapper = &info.message_wrapper; - let signal = &info.extern_signal_ty; - let error_ty = &info.extern_error_ty; - let support_crate = info.support_crate_name(); - - let mut ts = quote! { - /// A context type that is given to the [`Subsystem`] upon spawning. - /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s - /// or to spawn it's [`SubsystemJob`]s. - /// - /// [`Overseer`]: struct.Overseer.html - /// [`Subsystem`]: trait.Subsystem.html - /// [`SubsystemJob`]: trait.SubsystemJob.html - #[derive(Debug)] - #[allow(missing_docs)] - pub struct #subsystem_ctx_name { - signals: #support_crate ::metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages< M >, - to_subsystems: #subsystem_sender_name < ::OutgoingMessages >, - to_overseer: #support_crate ::metered::UnboundedMeteredSender< - #support_crate ::ToOverseer - >, - signals_received: SignalsReceived, - pending_incoming: Option<(usize, M)>, - name: &'static str - } - - impl #subsystem_ctx_name - where - M: AssociateOutgoing + Send + 'static, - { - /// Create a new context. - fn new( - signals: #support_crate ::metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages< M >, - to_subsystems: ChannelsOut, - to_overseer: #support_crate ::metered::UnboundedMeteredSender<#support_crate:: ToOverseer>, - name: &'static str - ) -> Self { - let signals_received = SignalsReceived::default(); - #subsystem_ctx_name :: { - signals, - messages, - to_subsystems: #subsystem_sender_name :: < ::OutgoingMessages > { - channels: to_subsystems, - signals_received: signals_received.clone(), - _phantom: ::core::marker::PhantomData::default(), - }, - to_overseer, - signals_received, - pending_incoming: None, - name - } - } - - fn name(&self) -> &'static str { - self.name - } - } - }; - // impl the subsystem context trait ts.extend(quote! { - #[#support_crate ::async_trait] - impl #support_crate ::SubsystemContext for #subsystem_ctx_name + impl #support_crate ::SubsystemContext for #subsystem_ctx_name < #consumes > where - M: AssociateOutgoing, - #all_messages_wrapper: From, - #subsystem_sender_name< - ::OutgoingMessages - >: - #support_crate ::SubsystemSender< - ::OutgoingMessages - >, + #where_clause { - type Message = M; + type Message = #consumes; type Signal = #signal; - type OutgoingMessages = ::OutgoingMessages; - // type AllMessages = #all_messages_wrapper; - type Sender = #subsystem_sender_name < ::OutgoingMessages >; + type OutgoingMessages = #outgoing_wrapper; + type Sender = #subsystem_sender_name < #outgoing_wrapper >; type Error = #error_ty; - async fn try_recv(&mut self) -> ::std::result::Result>, ()> { + async fn try_recv(&mut self) -> ::std::result::Result>, ()> { match #support_crate ::poll!(self.recv()) { #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), #support_crate ::Poll::Pending => Ok(None), } } - async fn recv(&mut self) -> ::std::result::Result, #error_ty> { + async fn recv(&mut self) -> ::std::result::Result, #error_ty> { loop { // If we have a message pending an overseer signal, we only poll for signals // in the meantime. @@ -748,6 +552,76 @@ pub(crate) fn impl_subsystem_context( } } }); + ts +} + +/// Implement a builder pattern for the `Overseer`-type, +/// which acts as the gateway to constructing the overseer. +pub(crate) fn impl_subsystem_context( + info: &OverseerInfo, + subsystem_sender_name: &Ident, + subsystem_ctx_name: &Ident, +) -> TokenStream { + let signal = &info.extern_signal_ty; + let support_crate = info.support_crate_name(); + + let ts = quote! { + /// A context type that is given to the [`Subsystem`] upon spawning. + /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s + /// or to spawn it's [`SubsystemJob`]s. + /// + /// [`Overseer`]: struct.Overseer.html + /// [`Subsystem`]: trait.Subsystem.html + /// [`SubsystemJob`]: trait.SubsystemJob.html + #[derive(Debug)] + #[allow(missing_docs)] + pub struct #subsystem_ctx_name { + signals: #support_crate ::metered::MeteredReceiver< #signal >, + messages: SubsystemIncomingMessages< M >, + to_subsystems: #subsystem_sender_name < ::OutgoingMessages >, + to_overseer: #support_crate ::metered::UnboundedMeteredSender< + #support_crate ::ToOverseer + >, + signals_received: SignalsReceived, + pending_incoming: Option<(usize, M)>, + name: &'static str + } + + impl #subsystem_ctx_name + where + M: AssociateOutgoing + Send + 'static, + { + /// Create a new context. + fn new( + signals: #support_crate ::metered::MeteredReceiver< #signal >, + messages: SubsystemIncomingMessages< M >, + to_subsystems: ChannelsOut, + to_overseer: #support_crate ::metered::UnboundedMeteredSender<#support_crate:: ToOverseer>, + name: &'static str + ) -> Self { + let signals_received = SignalsReceived::default(); + #subsystem_ctx_name :: { + signals, + messages, + to_subsystems: #subsystem_sender_name :: < ::OutgoingMessages > { + channels: to_subsystems, + signals_received: signals_received.clone(), + _phantom: ::core::marker::PhantomData::default(), + }, + to_overseer, + signals_received, + pending_incoming: None, + name + } + } + + fn name(&self) -> &'static str { + self.name + } + } + }; + + ts } From d284bc7992bde26f2310303d39713fdfc86a0c4c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 10:02:09 +0200 Subject: [PATCH 069/139] chore: move solo up --- node/overseer/overseer-gen/examples/solo.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index eb27d9b53eaa..95e849ba17f2 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -5,6 +5,14 @@ mod misc; pub use self::misc::*; + +#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] +struct Solo { + #[subsystem(no_dispatch, consumes: Plinko, sends: [MsgStrukt])] + goblin_tower: GoblinTower, +} + + #[derive(Default)] pub struct GoblinTower; @@ -28,12 +36,6 @@ where } -#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] -struct Solo { - #[subsystem(no_dispatch, consumes: Plinko, sends: [MsgStrukt])] - goblin_tower: GoblinTower, -} - fn main() { let (overseer, _handle): (Solo<_, f64>, _) = Solo::builder() .goblin_tower(GoblinTower::default()) From 11f8e8206729c11a806060b6d7bb1ec162ae79bd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 10:02:21 +0200 Subject: [PATCH 070/139] better trait bounds --- .../proc-macro/src/impl_builder.rs | 57 ++++++++++++++----- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 371bd7aa4de2..ce3ae17d000a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -83,7 +83,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { // Helpers to use within quote! macros let spawner_where_clause: syn::TypeParam = parse_quote! { - S: #support_crate ::SpawnNamed + Send + S: #support_crate ::SpawnNamed + Send, }; // Field names and real types @@ -130,19 +130,28 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { // see the loop below. let to_keep_subsystem_name = recollect_without_idx(&subsystem_name[..], idx); + let subsystem_sender_trait = format_ident!("{}SenderTrait", field_type); + let _subsystem_ctx_trait = format_ident!("{}ContextTrait", field_type); + + let builder_where_clause = quote!{ + #field_type : #support_crate::Subsystem< #subsystem_ctx_name< #subsystem_consumes >, #error_ty>, + < #subsystem_ctx_name < #subsystem_consumes > as #support_crate :: SubsystemContext>::Sender: + #subsystem_sender_trait, + }; + // Create the field init `fn` quote! { impl #builder where - #field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>, + #builder_where_clause { /// Specify the subsystem in the builder directly pub fn #field_name (self, var: #field_type ) -> #builder { #builder { - #field_name: Init::<#field_type>::Value(var), + #field_name: Init::< #field_type >::Value(var), #( #to_keep_subsystem_name: self. #to_keep_subsystem_name, )* @@ -158,7 +167,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { /// Specify the the initialization function for a subsystem pub fn #field_name_with<'a, F>(self, subsystem_init_fn: F ) -> #builder - where + where F: 'static + FnOnce(#handle) -> ::std::result::Result<#field_type, #error_ty>, { @@ -185,7 +194,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { impl #builder where - #field_type : Subsystem<#subsystem_ctx_name<#subsystem_consumes>, #error_ty>, + #builder_where_clause { /// Replace a subsystem by another implementation for the /// consumable message type. @@ -301,6 +310,27 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { S, #( #baggage_generic_ty, )* #( #subsystem_generics, )* }; + let builder_where_clause = info.subsystems().iter().map(|ssf| { + let field_type = &ssf.generic; + let consumes = &ssf.message_to_consume; + let subsystem_sender_trait = format_ident!("{}SenderTrait", ssf.generic); + let subsystem_ctx_trait = format_ident!("{}ContextTrait", ssf.generic); + let outgoing_wrapper = format_ident!("{}OutgoingMessages", ssf.generic); + quote!{ + #field_type: + #support_crate::Subsystem< #support_crate :: #subsystem_ctx_name, #error_ty>, + #support_crate :: #subsystem_ctx_name< #consumes >: + #subsystem_ctx_trait, + <#support_crate :: #subsystem_ctx_name as #subsystem_ctx_trait>::Sender: + #subsystem_sender_trait, + } + }).fold(TokenStream::new(), |mut ts, addendum| { + ts.extend(addendum); + ts + }); + + + let mut ts = quote! { /// Convenience alias. type SubsystemInitFn = Box ::std::result::Result >; @@ -332,14 +362,15 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { } } - impl #overseer_name where #spawner_where_clause { + impl #overseer_name + where + #spawner_where_clause + { /// Create a new overseer utilizing the builder. pub fn builder< #( #subsystem_generics),* >() -> - #builder #(, Missing<#field_type> )* > + #builder #(, Missing< #field_type > )* > where - #( - #subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>, - )* + #builder_where_clause { #builder :: new() } @@ -502,10 +533,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { // A builder specialization where all fields are set impl<#initialized_builder_generics> #initialized_builder<#initialized_builder_generics> where - #spawner_where_clause, - #( - #subsystem_generics : Subsystem<#subsystem_ctx_name< #consumes >, #error_ty>, - )* + #spawner_where_clause + #builder_where_clause { /// Complete the construction and create the overseer type. pub fn build(self) From 1ef4b79f35a8301e46fbeaf75809b26b9a60ac37 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 13:04:50 +0200 Subject: [PATCH 071/139] introduce `#[subsystem]` annotation FIXME Error is hardcoded to Yikes --- node/overseer/overseer-gen/examples/duo.rs | 18 ++-- node/overseer/overseer-gen/examples/solo.rs | 14 ++- .../proc-macro/src/impl_builder.rs | 21 ++--- .../proc-macro/src/impl_channels_out.rs | 14 ++- .../proc-macro/src/impl_overseer.rs | 5 ++ .../proc-macro/src/impl_subsystem.rs | 17 +++- .../overseer-gen/proc-macro/src/lib.rs | 87 ++++++++++++++++++- node/overseer/overseer-gen/src/lib.rs | 17 ++-- 8 files changed, 152 insertions(+), 41 deletions(-) diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs index 962a25c22c62..bf6bae9689f3 100644 --- a/node/overseer/overseer-gen/examples/duo.rs +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -1,7 +1,7 @@ //! A dummy to be used with cargo expand use polkadot_node_network_protocol::WrongVariant; -use polkadot_overseer_gen::{SpawnNamed, *}; +use polkadot_overseer_gen::{SpawnNamed, self as overseer, *}; use std::collections::HashMap; mod misc; @@ -11,11 +11,10 @@ pub use self::misc::*; #[derive(Default)] pub struct AwesomeSubSys; -impl overseer::Subsystem for AwesomeSubSys -where - Ctx: AwesomeSubSysContextTrait, +#[overseer::subsystem] +impl AwesomeSubSys { - fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { + fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( "AwesomeSubsys", @@ -31,12 +30,11 @@ where #[derive(Default)] pub struct GoblinTower; -impl overseer::Subsystem for GoblinTower -where - Ctx: GoblinTowerContextTrait, - GoblinTowerContextTrait, + +#[overseer::subsystem] +impl GoblinTower { - fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { + fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( "GoblinTower", diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index 95e849ba17f2..8b26241b9943 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -1,6 +1,7 @@ //! A minimal demo to be used with cargo expand. use polkadot_overseer_gen::{SpawnNamed, self as overseer, *}; +use overseer::SubsystemSender as _; mod misc; pub use self::misc::*; @@ -16,13 +17,10 @@ struct Solo { #[derive(Default)] pub struct GoblinTower; -impl overseer::Subsystem for GoblinTower -where - Ctx: GoblinTowerContextTrait, - ::Sender: GoblinTowerSenderTrait, - ::Sender: GoblinTowerSenderTrait, -{ - fn start(self, mut ctx: Ctx) -> SpawnedSubsystem { + +#[overseer::subsystem] +impl GoblinTower { + fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( "GoblinTower", @@ -37,7 +35,7 @@ where fn main() { - let (overseer, _handle): (Solo<_, f64>, _) = Solo::builder() + let (overseer, _handle): (Solo<_>, _) = Solo::builder() .goblin_tower(GoblinTower::default()) .spawner(DummySpawner) .build() diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index ce3ae17d000a..b5c233baf513 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -83,7 +83,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { // Helpers to use within quote! macros let spawner_where_clause: syn::TypeParam = parse_quote! { - S: #support_crate ::SpawnNamed + Send, + S: #support_crate ::SpawnNamed + Send }; // Field names and real types @@ -318,11 +318,11 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { let outgoing_wrapper = format_ident!("{}OutgoingMessages", ssf.generic); quote!{ #field_type: - #support_crate::Subsystem< #support_crate :: #subsystem_ctx_name, #error_ty>, - #support_crate :: #subsystem_ctx_name< #consumes >: - #subsystem_ctx_trait, - <#support_crate :: #subsystem_ctx_name as #subsystem_ctx_trait>::Sender: + #support_crate::Subsystem< #subsystem_ctx_name < #consumes>, #error_ty>, + <#subsystem_ctx_name< #consumes > as #subsystem_ctx_trait>::Sender: #subsystem_sender_trait, + #subsystem_ctx_name< #consumes >: + #subsystem_ctx_trait, } }).fold(TokenStream::new(), |mut ts, addendum| { ts.extend(addendum); @@ -362,9 +362,9 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { } } - impl #overseer_name + impl #overseer_name where - #spawner_where_clause + #spawner_where_clause, { /// Create a new overseer utilizing the builder. pub fn builder< #( #subsystem_generics),* >() -> @@ -429,7 +429,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { ts.extend(quote!{ /// Builder pattern to create compile time safe construction path. - pub struct #builder { + pub struct #builder + { #( #subsystem_name: #subsystem_passthrough_state_generics, )* @@ -476,7 +477,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { impl #builder, #( #subsystem_passthrough_state_generics, )* #( #baggage_passthrough_state_generics, )*> where - #spawner_where_clause + #spawner_where_clause, { /// The `spawner` to use for spawning tasks. pub fn spawner(self, spawner: S) -> #builder< @@ -533,7 +534,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { // A builder specialization where all fields are set impl<#initialized_builder_generics> #initialized_builder<#initialized_builder_generics> where - #spawner_where_clause + #spawner_where_clause, #builder_where_clause { /// Complete the construction and create the overseer type. diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index f61071abc8e1..2ef1c20565a6 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -79,6 +79,12 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), + + // And everything that's not WIP but no subsystem consumes it + unused_msg => { + #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); + Ok(()) + } }; if let Err(subsystem_name) = res { @@ -110,7 +116,13 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), )* // dummy message type - #message_wrapper :: Empty => Ok(()) + #message_wrapper :: Empty => Ok(()), + + // And everything that's not WIP but no subsystem consumes it + unused_msg => { + #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); + Ok(()) + } }; if let Err(subsystem_name) = res { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 4721eb5373e1..113bbdcec421 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -148,6 +148,11 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> proc_macro2::TokenStr #message_wrapper :: #unconsumes_variant ( _ ) => {} )* #message_wrapper :: Empty => {} + + // And everything that's not WIP but no subsystem consumes it + unused_msg => { + #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); + } } Ok(()) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs index 4c7540ae6976..2d1f28332e5a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs @@ -119,6 +119,11 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { #outgoing_wrapper :: #outgoing_variant ( msg ) => #all_messages_wrapper :: #subsystem_generic ( msg ), )* #outgoing_wrapper :: Empty => #all_messages_wrapper :: Empty, + // And everything that's not WIP but no subsystem consumes it + unused_msg => { + #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); + #all_messages_wrapper :: Empty + } } } } @@ -405,9 +410,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( #all_messages_wrapper: From< #outgoing_wrapper >, #all_messages_wrapper: From< #consumes >, #support_crate :: #outgoing_wrapper: #( From< #outgoing > )+*, - ::Sender: - #subsystem_sender_trait - + #acc_sender_trait_bounds, + }; ts.extend(quote! { @@ -421,9 +424,12 @@ pub(crate) fn impl_per_subsystem_helper_traits( > where #where_clause + ::Sender: + #subsystem_sender_trait + + #acc_sender_trait_bounds, { /// Sender. - type Sender: #subsystem_sender_trait + #acc_sender_trait_bounds; + type Sender: #subsystem_sender_trait; } impl #subsystem_ctx_trait for T @@ -436,6 +442,9 @@ pub(crate) fn impl_per_subsystem_helper_traits( Error = #error_ty, >, #where_clause + ::Sender: + #subsystem_sender_trait + + #acc_sender_trait_bounds, { type Sender = ::Sender; } diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index e198a0152d7f..e1868fe465d1 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -17,8 +17,8 @@ #![deny(unused_crate_dependencies)] use proc_macro2::{Ident, Span, TokenStream}; -use quote::{quote, ToTokens}; -use syn::{parse2, Result}; +use quote::{quote, ToTokens, format_ident}; +use syn::{parse2, spanned::Spanned, Result, punctuated::Punctuated, parse_quote, Token}; mod impl_builder; mod impl_channels_out; @@ -51,6 +51,18 @@ pub fn overlord( .into() } +#[proc_macro_attribute] +pub fn subsystem( + attr: proc_macro::TokenStream, + item: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + let attr: TokenStream = attr.into(); + let item: TokenStream = item.into(); + impl_subsystem_gen(attr, item) + .unwrap_or_else(|err| err.to_compile_error()) + .into() +} + pub(crate) fn impl_overseer_gen( attr: TokenStream, orig: TokenStream, @@ -106,3 +118,74 @@ pub(crate) fn impl_overseer_gen( Ok(ts) } + + +pub(crate) fn impl_subsystem_gen( + attr: TokenStream, + orig: TokenStream, +) -> Result { + if !attr.is_empty() { + return Err( + syn::Error::new( + attr.span(), + "Doesn't take any arguments at this time" + ) + ); + } + let mut struktured_impl: syn::ItemImpl = parse2(orig)?; + + let support_crate = if cfg!(test) { + quote! {crate} + } else { + use proc_macro_crate::{crate_name, FoundCrate}; + let crate_name = crate_name("polkadot-overseer-gen") + .expect("Support crate polkadot-overseer-gen is present in `Cargo.toml`. qed"); + match crate_name { + FoundCrate::Itself => quote! {crate}, + FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(), + } + }; + + let me = match &*struktured_impl.self_ty { + syn::Type::Path(ref path) => { + dbg!(path).path.segments.last().expect("Must be ident. FIXME bail properly").ident.clone() + }, + _ => { + return Err( + syn::Error::new( + attr.span(), + "Doesn't take any arguments at this time" + ) + ) + } + }; + let subsystem_ctx_trait = format_ident!("{}ContextTrait", me); + let subsystem_sender_trait = format_ident!("{}SenderTrait", me); + let extra_where_predicates: Punctuated = parse_quote!{ + Context: #subsystem_ctx_trait, + Context: #support_crate::SubsystemContext, + ::Sender: #subsystem_sender_trait, + ::Sender: #subsystem_sender_trait, + }; + let error_ty = quote!{ Yikes }; // FIXME + struktured_impl.trait_.replace( + ( + None, + parse_quote!{ #support_crate::Subsystem}, + syn::token::For::default(), + ) + ); + let where_clause = struktured_impl.generics.make_where_clause(); + where_clause.predicates.extend(extra_where_predicates); + + + let ts = expander::Expander::new(format!("subsystem-{}-expansion", subsystem_ctx_trait.to_string().to_lowercase())) + .add_comment("Generated overseer code by `#[subsystem(..)]`".to_owned()) + .dry(!cfg!(feature = "expand")) + .verbose(true) + .fmt(expander::Edition::_2021) + .write_to_out_dir(struktured_impl.into_token_stream()) + .expect("Expander does not fail due to IO in OUT_DIR. qed"); + + Ok(ts) +} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index e1908528b9e9..159c26ad7e96 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -62,6 +62,8 @@ pub use polkadot_overseer_gen_proc_macro::overlord; +pub use polkadot_overseer_gen_proc_macro::subsystem; + #[doc(hidden)] pub use gum; #[doc(hidden)] @@ -372,7 +374,7 @@ pub trait SubsystemContext: Send + 'static { // type AllMessages: From + From + std::fmt::Debug + Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: Clone + SubsystemSender + Send + 'static; + type Sender: Clone + Send + 'static; /// The error type. type Error: ::std::error::Error + ::std::convert::From + Sync + Send + 'static; @@ -404,9 +406,10 @@ pub trait SubsystemContext: Send + 'static { async fn send_message(&mut self, msg: T) where Self::OutgoingMessages: From + Send, + // Self::Sender: SubsystemSender, T: Send, { - self.sender().send_message(::from(msg)).await + // self.sender().send_message(::from(msg)).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. @@ -414,13 +417,14 @@ pub trait SubsystemContext: Send + 'static { async fn send_messages(&mut self, msgs: I) where Self::OutgoingMessages: From + Send, + // Self::Sender: SubsystemSender, I: IntoIterator + Send, I::IntoIter: Send, T: Send, { - self.sender() - .send_messages(msgs.into_iter().map(::from)) - .await + // self.sender() + // .send_messages(msgs.into_iter().map(::from)) + // .await } /// Send a message using the unbounded connection. @@ -428,9 +432,10 @@ pub trait SubsystemContext: Send + 'static { fn send_unbounded_message(&mut self, msg: X) where Self::OutgoingMessages: From + Send, + Self::Sender: SubsystemSender, X: Send, { - self.sender().send_unbounded_message(::from(msg)) + // self.sender().send_unbounded_message(::from(msg)) } /// Obtain the sender. From 899f1234e814a2cda8a0a5806a6eca1d29cb6a87 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 13:45:38 +0200 Subject: [PATCH 072/139] parse attr of `#[subsystem]` --- node/overseer/overseer-gen/examples/duo.rs | 14 +-- node/overseer/overseer-gen/examples/misc.rs | 2 - node/overseer/overseer-gen/examples/solo.rs | 9 +- .../proc-macro/src/impl_builder.rs | 44 +++---- ...system.rs => impl_subsystem_ctx_sender.rs} | 2 - .../overseer-gen/proc-macro/src/lib.rs | 73 +++++------- .../overseer-gen/proc-macro/src/parse/mod.rs | 22 +++- .../{parse_attr.rs => parse_overseer_attr.rs} | 18 +-- ...rse_struct.rs => parse_overseer_struct.rs} | 0 .../src/parse/parse_subsystem_attr.rs | 112 ++++++++++++++++++ .../proc-macro/src/parse/tests.rs | 8 +- .../overseer-gen/proc-macro/src/tests.rs | 8 +- 12 files changed, 205 insertions(+), 107 deletions(-) rename node/overseer/overseer-gen/proc-macro/src/{impl_subsystem.rs => impl_subsystem_ctx_sender.rs} (99%) rename node/overseer/overseer-gen/proc-macro/src/parse/{parse_attr.rs => parse_overseer_attr.rs} (95%) rename node/overseer/overseer-gen/proc-macro/src/parse/{parse_struct.rs => parse_overseer_struct.rs} (100%) create mode 100644 node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs index bf6bae9689f3..3800bc08395c 100644 --- a/node/overseer/overseer-gen/examples/duo.rs +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -1,7 +1,7 @@ //! A dummy to be used with cargo expand use polkadot_node_network_protocol::WrongVariant; -use polkadot_overseer_gen::{SpawnNamed, self as overseer, *}; +use polkadot_overseer_gen::{self as overseer, SpawnNamed, *}; use std::collections::HashMap; mod misc; @@ -11,9 +11,8 @@ pub use self::misc::*; #[derive(Default)] pub struct AwesomeSubSys; -#[overseer::subsystem] -impl AwesomeSubSys -{ +#[overseer::subsystem(error=Yikes)] +impl AwesomeSubSys { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( @@ -30,10 +29,8 @@ impl AwesomeSubSys #[derive(Default)] pub struct GoblinTower; - -#[overseer::subsystem] -impl GoblinTower -{ +#[overseer::subsystem(error=Yikes)] +impl GoblinTower { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( @@ -60,6 +57,7 @@ struct Duo { i_like_hash: HashMap, } +#[test] fn main() { let (overseer, _handle): (Duo<_, f64>, _) = Duo::builder() .sub0(AwesomeSubSys::default()) diff --git a/node/overseer/overseer-gen/examples/misc.rs b/node/overseer/overseer-gen/examples/misc.rs index 85b50b9a482b..19b85f3a2f8c 100644 --- a/node/overseer/overseer-gen/examples/misc.rs +++ b/node/overseer/overseer-gen/examples/misc.rs @@ -23,8 +23,6 @@ impl SpawnNamed for DummySpawner { } } - - /// A signal sent by the overseer. #[derive(Debug, Clone)] pub struct SigSigSig; diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index 8b26241b9943..9997531fa798 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -1,24 +1,21 @@ //! A minimal demo to be used with cargo expand. -use polkadot_overseer_gen::{SpawnNamed, self as overseer, *}; use overseer::SubsystemSender as _; +use polkadot_overseer_gen::{self as overseer, SpawnNamed, *}; mod misc; pub use self::misc::*; - #[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] struct Solo { #[subsystem(no_dispatch, consumes: Plinko, sends: [MsgStrukt])] goblin_tower: GoblinTower, } - #[derive(Default)] pub struct GoblinTower; - -#[overseer::subsystem] +#[overseer::subsystem(error=Yikes)] impl GoblinTower { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); @@ -33,7 +30,7 @@ impl GoblinTower { } } - +#[test] fn main() { let (overseer, _handle): (Solo<_>, _) = Solo::builder() .goblin_tower(GoblinTower::default()) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index b5c233baf513..ceb9faf0ca39 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -310,26 +310,27 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { S, #( #baggage_generic_ty, )* #( #subsystem_generics, )* }; - let builder_where_clause = info.subsystems().iter().map(|ssf| { - let field_type = &ssf.generic; - let consumes = &ssf.message_to_consume; - let subsystem_sender_trait = format_ident!("{}SenderTrait", ssf.generic); - let subsystem_ctx_trait = format_ident!("{}ContextTrait", ssf.generic); - let outgoing_wrapper = format_ident!("{}OutgoingMessages", ssf.generic); - quote!{ - #field_type: - #support_crate::Subsystem< #subsystem_ctx_name < #consumes>, #error_ty>, - <#subsystem_ctx_name< #consumes > as #subsystem_ctx_trait>::Sender: - #subsystem_sender_trait, - #subsystem_ctx_name< #consumes >: - #subsystem_ctx_trait, - } - }).fold(TokenStream::new(), |mut ts, addendum| { - ts.extend(addendum); - ts - }); - - + let builder_where_clause = info + .subsystems() + .iter() + .map(|ssf| { + let field_type = &ssf.generic; + let consumes = &ssf.message_to_consume; + let subsystem_sender_trait = format_ident!("{}SenderTrait", ssf.generic); + let subsystem_ctx_trait = format_ident!("{}ContextTrait", ssf.generic); + quote! { + #field_type: + #support_crate::Subsystem< #subsystem_ctx_name < #consumes>, #error_ty>, + <#subsystem_ctx_name< #consumes > as #subsystem_ctx_trait>::Sender: + #subsystem_sender_trait, + #subsystem_ctx_name< #consumes >: + #subsystem_ctx_trait, + } + }) + .fold(TokenStream::new(), |mut ts, addendum| { + ts.extend(addendum); + ts + }); let mut ts = quote! { /// Convenience alias. @@ -523,7 +524,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { }); // Create the string literals for spawn. - let subsystem_name_str_literal = subsystem_name.iter() + let subsystem_name_str_literal = subsystem_name + .iter() .map(|ident| proc_macro2::Literal::string(ident.to_string().replace("_", "-").as_str())) .collect::>(); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs similarity index 99% rename from node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs rename to node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index 2d1f28332e5a..b3e2889beb32 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -630,7 +630,5 @@ pub(crate) fn impl_subsystem_context( } }; - - ts } diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index e1868fe465d1..050e9804241d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -17,15 +17,15 @@ #![deny(unused_crate_dependencies)] use proc_macro2::{Ident, Span, TokenStream}; -use quote::{quote, ToTokens, format_ident}; -use syn::{parse2, spanned::Spanned, Result, punctuated::Punctuated, parse_quote, Token}; +use quote::{format_ident, quote, ToTokens}; +use syn::{parse2, parse_quote, punctuated::Punctuated, spanned::Spanned, Result}; mod impl_builder; mod impl_channels_out; mod impl_dispatch; mod impl_message_wrapper; mod impl_overseer; -mod impl_subsystem; +mod impl_subsystem_ctx_sender; mod parse; use impl_builder::*; @@ -33,7 +33,7 @@ use impl_channels_out::*; use impl_dispatch::*; use impl_message_wrapper::*; use impl_overseer::*; -use impl_subsystem::*; +use impl_subsystem_ctx_sender::*; use parse::*; #[cfg(test)] @@ -67,7 +67,7 @@ pub(crate) fn impl_overseer_gen( attr: TokenStream, orig: TokenStream, ) -> Result { - let args: AttrArgs = parse2(attr)?; + let args: OverseerAttrArgs = parse2(attr)?; let message_wrapper = args.message_wrapper; let of: OverseerGuts = parse2(orig)?; @@ -119,19 +119,12 @@ pub(crate) fn impl_overseer_gen( Ok(ts) } - pub(crate) fn impl_subsystem_gen( attr: TokenStream, orig: TokenStream, ) -> Result { - if !attr.is_empty() { - return Err( - syn::Error::new( - attr.span(), - "Doesn't take any arguments at this time" - ) - ); - } + let error_ty = parse2::(attr.clone())?.extern_error_ty; + let mut struktured_impl: syn::ItemImpl = parse2(orig)?; let support_crate = if cfg!(test) { @@ -147,45 +140,41 @@ pub(crate) fn impl_subsystem_gen( }; let me = match &*struktured_impl.self_ty { - syn::Type::Path(ref path) => { - dbg!(path).path.segments.last().expect("Must be ident. FIXME bail properly").ident.clone() - }, - _ => { - return Err( - syn::Error::new( - attr.span(), - "Doesn't take any arguments at this time" - ) - ) - } + syn::Type::Path(ref path) => dbg!(path) + .path + .segments + .last() + .expect("Must be ident. FIXME bail properly") + .ident + .clone(), + _ => return Err(syn::Error::new(attr.span(), "Doesn't take any arguments at this time")), }; let subsystem_ctx_trait = format_ident!("{}ContextTrait", me); let subsystem_sender_trait = format_ident!("{}SenderTrait", me); - let extra_where_predicates: Punctuated = parse_quote!{ + let extra_where_predicates: Punctuated = parse_quote! { Context: #subsystem_ctx_trait, Context: #support_crate::SubsystemContext, ::Sender: #subsystem_sender_trait, ::Sender: #subsystem_sender_trait, }; - let error_ty = quote!{ Yikes }; // FIXME - struktured_impl.trait_.replace( - ( - None, - parse_quote!{ #support_crate::Subsystem}, - syn::token::For::default(), - ) - ); + struktured_impl.trait_.replace(( + None, + parse_quote! { + #support_crate::Subsystem + }, + syn::token::For::default(), + )); let where_clause = struktured_impl.generics.make_where_clause(); where_clause.predicates.extend(extra_where_predicates); - - let ts = expander::Expander::new(format!("subsystem-{}-expansion", subsystem_ctx_trait.to_string().to_lowercase())) - .add_comment("Generated overseer code by `#[subsystem(..)]`".to_owned()) - .dry(!cfg!(feature = "expand")) - .verbose(true) - .fmt(expander::Edition::_2021) - .write_to_out_dir(struktured_impl.into_token_stream()) - .expect("Expander does not fail due to IO in OUT_DIR. qed"); + let ts = + expander::Expander::new(format!("subsystem-{}-expansion", me.to_string().to_lowercase())) + .add_comment("Generated overseer code by `#[subsystem(..)]`".to_owned()) + .dry(!cfg!(feature = "expand")) + .verbose(true) + .fmt(expander::Edition::_2021) + .write_to_out_dir(struktured_impl.into_token_stream()) + .expect("Expander does not fail due to IO in OUT_DIR. qed"); Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs index a99232458c46..42ef44a02336 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs @@ -14,11 +14,25 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -mod parse_attr; -mod parse_struct; +mod kw { + syn::custom_keyword!(event); + syn::custom_keyword!(signal); + syn::custom_keyword!(error); + syn::custom_keyword!(network); + syn::custom_keyword!(outgoing); + syn::custom_keyword!(gen); + syn::custom_keyword!(signal_capacity); + syn::custom_keyword!(message_capacity); +} + +mod parse_overseer_attr; +mod parse_overseer_struct; + +mod parse_subsystem_attr; #[cfg(test)] mod tests; -pub(crate) use parse_attr::*; -pub(crate) use parse_struct::*; +pub(crate) use self::{parse_overseer_attr::*, parse_overseer_struct::*}; + +pub(crate) use self::parse_subsystem_attr::*; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs similarity index 95% rename from node/overseer/overseer-gen/proc-macro/src/parse/parse_attr.rs rename to node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs index e688f84f2e37..9bd9a5ab6212 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use super::kw; use proc_macro2::Span; use quote::{quote, ToTokens}; use std::collections::{hash_map::RandomState, HashMap}; @@ -24,17 +25,6 @@ use syn::{ Error, Ident, LitInt, Path, Result, Token, }; -mod kw { - syn::custom_keyword!(event); - syn::custom_keyword!(signal); - syn::custom_keyword!(error); - syn::custom_keyword!(network); - syn::custom_keyword!(outgoing); - syn::custom_keyword!(gen); - syn::custom_keyword!(signal_capacity); - syn::custom_keyword!(message_capacity); -} - #[derive(Clone, Debug)] enum OverseerAttrItem { ExternEventType { tag: kw::event, eq_token: Token![=], value: Path }, @@ -138,7 +128,7 @@ impl Parse for OverseerAttrItem { /// Attribute arguments #[derive(Clone, Debug)] -pub(crate) struct AttrArgs { +pub(crate) struct OverseerAttrArgs { pub(crate) message_wrapper: Ident, pub(crate) extern_event_ty: Path, pub(crate) extern_signal_ty: Path, @@ -170,7 +160,7 @@ macro_rules! extract_variant { }; } -impl Parse for AttrArgs { +impl Parse for OverseerAttrArgs { fn parse(input: &ParseBuffer) -> Result { let items: Punctuated = input.parse_terminated(OverseerAttrItem::parse)?; @@ -203,7 +193,7 @@ impl Parse for AttrArgs { let network = extract_variant!(unique, ExternNetworkType); let outgoing = extract_variant!(unique, OutgoingType); - Ok(AttrArgs { + Ok(OverseerAttrArgs { signal_channel_capacity, message_channel_capacity, extern_event_ty: event, diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs similarity index 100% rename from node/overseer/overseer-gen/proc-macro/src/parse/parse_struct.rs rename to node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs new file mode 100644 index 000000000000..a4cd96882909 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs @@ -0,0 +1,112 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::kw; +use proc_macro2::Span; +use quote::{quote, ToTokens}; +use std::collections::{hash_map::RandomState, HashMap}; +use syn::{ + parse::{Parse, ParseBuffer}, + punctuated::Punctuated, + spanned::Spanned, + Error, Path, Result, Token, +}; + +#[derive(Clone, Debug)] +enum SubsystemAttrItem { + ExternErrorType { tag: kw::error, eq_token: Token![=], value: Path }, +} + +impl ToTokens for SubsystemAttrItem { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + let ts = match self { + Self::ExternErrorType { tag, eq_token, value } => { + quote! { #tag #eq_token, #value } + }, + }; + tokens.extend(ts.into_iter()); + } +} + +impl Parse for SubsystemAttrItem { + fn parse(input: &ParseBuffer) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::error) { + Ok(SubsystemAttrItem::ExternErrorType { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } else { + Err(lookahead.error()) + } + } +} + +/// Attribute arguments +#[derive(Clone, Debug)] +pub(crate) struct SubsystemAttrArgs { + span: Span, + pub(crate) extern_error_ty: Path, +} + +impl Spanned for SubsystemAttrArgs { + fn span(&self) -> Span { + self.span.clone() + } +} + +macro_rules! extract_variant { + ($unique:expr, $variant:ident ; default = $fallback:expr) => { + extract_variant!($unique, $variant).unwrap_or_else(|| $fallback) + }; + ($unique:expr, $variant:ident ; err = $err:expr) => { + extract_variant!($unique, $variant).ok_or_else(|| Error::new(Span::call_site(), $err)) + }; + ($unique:expr, $variant:ident) => { + $unique.values().find_map(|item| { + match item { + SubsystemAttrItem::$variant { value, .. } => Some(value.clone()), + } + }) + }; +} + +impl Parse for SubsystemAttrArgs { + fn parse(input: &ParseBuffer) -> Result { + let span = input.span(); + let items: Punctuated = + input.parse_terminated(SubsystemAttrItem::parse)?; + + let mut unique = HashMap::< + std::mem::Discriminant, + SubsystemAttrItem, + RandomState, + >::default(); + for item in items { + if let Some(first) = unique.insert(std::mem::discriminant(&item), item.clone()) { + let mut e = Error::new( + item.span(), + format!("Duplicate definition of subsystem generation type found"), + ); + e.combine(Error::new(first.span(), "previously defined here.")); + return Err(e) + } + } + let error = extract_variant!(unique, ExternErrorType; err = "Must annotate the identical overseer error type via `error=..`.")?; + Ok(SubsystemAttrArgs { span, extern_error_ty: error }) + } +} diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs index 7032c2e7f9c9..b188b451ba80 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs @@ -25,11 +25,11 @@ mod attr { #[test] fn attr_full_works() { - let attr: AttrArgs = parse_quote! { + let attr: OverseerAttrArgs = parse_quote! { gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, error=OverseerError, }; - assert_matches!(attr, AttrArgs { + assert_matches!(attr, OverseerAttrArgs { message_channel_capacity, signal_channel_capacity, .. @@ -41,11 +41,11 @@ mod attr { #[test] fn attr_partial_works() { - let attr: AttrArgs = parse_quote! { + let attr: OverseerAttrArgs = parse_quote! { gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, error=OverseerError, }; - assert_matches!(attr, AttrArgs { + assert_matches!(attr, OverseerAttrArgs { message_channel_capacity: _, signal_channel_capacity: _, .. diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 40df210fb6f0..89803f3761a4 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -88,11 +88,11 @@ fn struct_parse_basic() { #[test] fn attr_full() { - let attr: AttrArgs = parse_quote! { + let attr: OverseerAttrArgs = parse_quote! { gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, error=OverseerError, }; - assert_matches!(attr, AttrArgs { + assert_matches!(attr, OverseerAttrArgs { message_channel_capacity, signal_channel_capacity, .. @@ -104,11 +104,11 @@ fn attr_full() { #[test] fn attr_partial() { - let attr: AttrArgs = parse_quote! { + let attr: OverseerAttrArgs = parse_quote! { gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, error=OverseerError, }; - assert_matches!(attr, AttrArgs { + assert_matches!(attr, OverseerAttrArgs { message_channel_capacity: _, signal_channel_capacity: _, .. From 580964c2732cd9ddb5eb8433a1ba6182614bc243 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 15:49:16 +0200 Subject: [PATCH 073/139] drop all dispatch_iter logic XXX INCOMPLETE MUST IMPL PROPER DISPATCH IN NETWORKING BRIDGE --- node/network/bridge/src/lib.rs | 37 +++++++--- node/network/bridge/src/tests.rs | 48 ------------- .../proc-macro/src/impl_dispatch.rs | 70 ------------------- .../overseer-gen/proc-macro/src/lib.rs | 3 - 4 files changed, 29 insertions(+), 129 deletions(-) delete mode 100644 node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 8e3c4f19c530..5689a8c76b25 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -1062,21 +1062,38 @@ fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { - // XXX FIXME XXX FIXME XXX - // for msg in AllMessages::dispatch_iter(event) { - // ctx.send_unbounded_message(msg); - // } + match event { + NetworkBridgeEvent::::BitfieldDistribution(bdm) => { + ctx.send_unbounded_message(bdm) + } + NetworkBridgeEvent::::StatementDistribution(sdm) => { + ctx.send_unbounded_message(sdm) + } + NetworkBridgeEvent::::ApprovalDistribution(adm) => { + ctx.send_unbounded_message(adm) + } + } } fn dispatch_collation_event_to_all_unbounded( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { - if let Some(msg) = event.focus().ok().map(CollatorProtocolMessage::NetworkBridgeUpdate) { - ctx.send_unbounded_message(msg); + + match event { + NetworkBridgeEvent::::BitfieldDistribution(bdm) => { + ctx.send_unbounded_message(bdm) + } + NetworkBridgeEvent::::StatementDistribution(sdm) => { + ctx.send_unbounded_message(sdm) + } + NetworkBridgeEvent::::ApprovalDistribution(adm) => { + ctx.send_unbounded_message(adm) + } } } + async fn dispatch_validation_events_to_all( events: I, ctx: &mut impl overseer::NetworkBridgeSenderTrait, @@ -1084,8 +1101,12 @@ async fn dispatch_validation_events_to_all( I: IntoIterator>, I::IntoIter: Send, { - // FIXME XXX XXX TODO XXX FIXME XXX - // ctx.send_messages(events.into_iter().flat_map(AllMessages::dispatch_iter)).await + for event in events { + sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; + sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; + sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; + sender.send_messages(event.focus().map(StatementDistributionMessage::from)).await; + } } async fn dispatch_collation_events_to_all( diff --git a/node/network/bridge/src/tests.rs b/node/network/bridge/src/tests.rs index 0697c0cfdea8..f9752900bbb5 100644 --- a/node/network/bridge/src/tests.rs +++ b/node/network/bridge/src/tests.rs @@ -1190,54 +1190,6 @@ fn send_messages_to_peers() { }); } -#[test] -fn spread_event_to_subsystems_is_up_to_date() { - // Number of subsystems expected to be interested in a network event, - // and hence the network event broadcasted to. - const EXPECTED_COUNT: usize = 4; - - let mut cnt = 0_usize; - for msg in AllMessages::dispatch_iter(NetworkBridgeEvent::PeerDisconnected(PeerId::random())) { - match msg { - AllMessages::Empty => unreachable!("Nobody cares about the dummy"), - AllMessages::CandidateValidation(_) => unreachable!("Not interested in network events"), - AllMessages::CandidateBacking(_) => unreachable!("Not interested in network events"), - AllMessages::ChainApi(_) => unreachable!("Not interested in network events"), - AllMessages::CollatorProtocol(_) => unreachable!("Not interested in network events"), - AllMessages::StatementDistribution(_) => { - cnt += 1; - }, - AllMessages::AvailabilityDistribution(_) => - unreachable!("Not interested in network events"), - AllMessages::AvailabilityRecovery(_) => - unreachable!("Not interested in network events"), - AllMessages::BitfieldDistribution(_) => { - cnt += 1; - }, - AllMessages::BitfieldSigning(_) => unreachable!("Not interested in network events"), - AllMessages::Provisioner(_) => unreachable!("Not interested in network events"), - AllMessages::RuntimeApi(_) => unreachable!("Not interested in network events"), - AllMessages::AvailabilityStore(_) => unreachable!("Not interested in network events"), - AllMessages::NetworkBridge(_) => unreachable!("Not interested in network events"), - AllMessages::CollationGeneration(_) => unreachable!("Not interested in network events"), - AllMessages::ApprovalVoting(_) => unreachable!("Not interested in network events"), - AllMessages::ApprovalDistribution(_) => { - cnt += 1; - }, - AllMessages::GossipSupport(_) => { - cnt += 1; - }, - AllMessages::DisputeCoordinator(_) => unreachable!("Not interested in network events"), - AllMessages::DisputeDistribution(_) => unreachable!("Not interested in network events"), - AllMessages::ChainSelection(_) => unreachable!("Not interested in network events"), - AllMessages::PvfChecker(_) => unreachable!("Not interested in network events"), - // Add variants here as needed, `{ cnt += 1; }` for those that need to be - // notified, `unreachable!()` for those that should not. - } - } - assert_eq!(cnt, EXPECTED_COUNT); -} - #[test] fn our_view_updates_decreasing_order_and_limited_to_max() { test_harness(done_syncing_oracle(), |test_harness| async move { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs deleted file mode 100644 index ba1ba5294ce3..000000000000 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use super::*; -use proc_macro2::{Ident, TokenStream}; -use quote::quote; -use syn::Path; - -pub(crate) fn impl_dispatch(info: &OverseerInfo) -> TokenStream { - let message_wrapper = &info.message_wrapper; - - let dispatchable_variant = info - .subsystems() - .into_iter() - .filter(|ssf| !ssf.no_dispatch) - .filter(|ssf| !ssf.wip) - .map(|ssf| ssf.generic.clone()) - .collect::>(); - - let dispatchable_message = info - .subsystems() - .into_iter() - .filter(|ssf| !ssf.no_dispatch) - .filter(|ssf| !ssf.wip) - .map(|ssf| ssf.message_to_consume.clone()) - .collect::>(); - - let mut ts = TokenStream::new(); - if let Some(extern_network_ty) = &info.extern_network_ty.clone() { - ts.extend(quote! { - impl #message_wrapper { - /// Generated dispatch iterator generator. - pub fn dispatch_iter(extern_msg: #extern_network_ty) -> impl Iterator + Send { - [ - #( - extern_msg - // focuses on a `NetworkBridgeEvent< protocol_v1::* >` - // TODO do not require this to be hardcoded, either externalize or ... - // - .focus() - .ok() - .map(|event| { - #message_wrapper :: #dispatchable_variant ( - // the inner type of the enum variant - #dispatchable_message :: from( event ) - ) - }), - )* - ] - .into_iter() - .filter_map(|x: Option<_>| x) - } - } - }); - } - ts -} diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 050e9804241d..b4eb2273a41b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -22,7 +22,6 @@ use syn::{parse2, parse_quote, punctuated::Punctuated, spanned::Spanned, Result} mod impl_builder; mod impl_channels_out; -mod impl_dispatch; mod impl_message_wrapper; mod impl_overseer; mod impl_subsystem_ctx_sender; @@ -30,7 +29,6 @@ mod parse; use impl_builder::*; use impl_channels_out::*; -use impl_dispatch::*; use impl_message_wrapper::*; use impl_overseer::*; use impl_subsystem_ctx_sender::*; @@ -106,7 +104,6 @@ pub(crate) fn impl_overseer_gen( additive.extend(impl_subsystem(&info)?); additive.extend(impl_message_wrapper_enum(&info)?); - additive.extend(impl_dispatch(&info)); let ts = expander::Expander::new("overlord-expansion") .add_comment("Generated overseer code by `#[overlord(..)]`".to_owned()) From 18a31f29fc350eaa518135568cd288d2781e160b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 15:50:15 +0200 Subject: [PATCH 074/139] feat: use new `#[subsystem]` everywhere --- node/collation-generation/src/lib.rs | 8 ++------ node/core/av-store/src/lib.rs | 8 ++------ node/core/candidate-validation/src/lib.rs | 8 ++------ node/core/dispute-coordinator/src/lib.rs | 8 ++------ node/network/approval-distribution/src/lib.rs | 8 ++------ node/network/availability-distribution/src/lib.rs | 8 ++------ node/network/availability-recovery/src/lib.rs | 8 ++------ node/network/bitfield-distribution/src/lib.rs | 8 ++------ node/network/collator-protocol/src/lib.rs | 8 ++------ 9 files changed, 18 insertions(+), 54 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index a61b6b98bcbc..ea0b459eb281 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -164,12 +164,8 @@ impl CollationGenerationSubsystem { } } -impl overseer::Subsystem for CollationGenerationSubsystem -where - Context: overseer::CollationGenerationContextTrait, - ::Sender: - overseer::CollationGenerationSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl CollationGenerationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { self.run(ctx).await; diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index af6a4f2df5a5..f1f2b5d432dc 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -519,12 +519,8 @@ impl KnownUnfinalizedBlocks { } } -impl overseer::Subsystem for AvailabilityStoreSubsystem -where - Context: overseer::AvailabilityStoreContextTrait, - ::Sender: - overseer::AvailabilityStoreSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl AvailabilityStoreSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(self, ctx).map(|_| Ok(())).boxed(); diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index da72e7b3b803..252657046eb0 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -93,12 +93,8 @@ impl CandidateValidationSubsystem { } } -impl overseer::Subsystem for CandidateValidationSubsystem -where - Context: overseer::CandidateValidationContextTrait, - ::Sender: - overseer::CandidateValidationSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl CandidateValidationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run( ctx, diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 8d371c572d42..21ae0cbcf002 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -124,12 +124,8 @@ impl Config { } } -impl overseer::Subsystem for DisputeCoordinatorSubsystem -where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl DisputeCoordinatorSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async { let backend = DbBackend::new(self.store.clone(), self.config.column_config()); diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 3e399932cf4f..f1d05d16ec03 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -1730,12 +1730,8 @@ impl ApprovalDistribution { } } -impl overseer::Subsystem for ApprovalDistribution -where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl ApprovalDistribution { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index 0bd5ead794bb..f3c9003c175f 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -68,12 +68,8 @@ pub struct IncomingRequestReceivers { pub chunk_req_receiver: IncomingRequestReceiver, } -impl overseer::Subsystem for AvailabilityDistributionSubsystem -where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl AvailabilityDistributionSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self .run(ctx) diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 55311c5653ae..e15ecf6b5399 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -806,12 +806,8 @@ impl Default for State { } } -impl Subsystem for AvailabilityRecoverySubsystem -where - Context: overseer::AvailabilityRecoveryContextTrait, - ::Sender: - overseer::AvailabilityRecoverySenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl AvailabilityRecoverySubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self .run(ctx) diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index ae607d493188..32469d660211 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -788,12 +788,8 @@ async fn send_tracked_gossip_message( .await; } -impl overseer::Subsystem for BitfieldDistribution -where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl BitfieldDistribution { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 860b9ce7a045..b748c1531f6b 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -113,12 +113,8 @@ impl CollatorProtocolSubsystem { } } -impl overseer::Subsystem for CollatorProtocolSubsystem -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, -{ +#[overseer::subsystem(error=SubsystemError)] +impl CollatorProtocolSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self .run(ctx) From 2c151d5ff113d081fe1095085aded23711a58c4c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 15:50:41 +0200 Subject: [PATCH 075/139] chore: fmt --- node/malus/src/interceptor.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/malus/src/interceptor.rs b/node/malus/src/interceptor.rs index 234d5afacf24..ae68ea9807c5 100644 --- a/node/malus/src/interceptor.rs +++ b/node/malus/src/interceptor.rs @@ -46,7 +46,10 @@ where } /// Modify outgoing messages. - fn intercept_outgoing(&self, msg: AssociateOutgoing::::OutgoingMessages) -> Option { + fn intercept_outgoing( + &self, + msg: AssociateOutgoing::OutgoingMessages, + ) -> Option { Some(msg) } } From 5c926a70c24d6d03e6bd8c9ac88d4a21409b328b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 15:51:04 +0200 Subject: [PATCH 076/139] fixup overseer gen Ctx sending foo Soon to be be deprecated, but N things at a time --- node/overseer/overseer-gen/src/lib.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 159c26ad7e96..815c8c2122c8 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -374,7 +374,7 @@ pub trait SubsystemContext: Send + 'static { // type AllMessages: From + From + std::fmt::Debug + Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: Clone + Send + 'static; + type Sender: Clone + Send + 'static + SubsystemSender; /// The error type. type Error: ::std::error::Error + ::std::convert::From + Sync + Send + 'static; @@ -406,10 +406,9 @@ pub trait SubsystemContext: Send + 'static { async fn send_message(&mut self, msg: T) where Self::OutgoingMessages: From + Send, - // Self::Sender: SubsystemSender, T: Send, { - // self.sender().send_message(::from(msg)).await + self.sender().send_message(::from(msg)).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. @@ -417,14 +416,13 @@ pub trait SubsystemContext: Send + 'static { async fn send_messages(&mut self, msgs: I) where Self::OutgoingMessages: From + Send, - // Self::Sender: SubsystemSender, I: IntoIterator + Send, I::IntoIter: Send, T: Send, { - // self.sender() - // .send_messages(msgs.into_iter().map(::from)) - // .await + self.sender() + .send_messages(msgs.into_iter().map(::from)) + .await } /// Send a message using the unbounded connection. @@ -432,10 +430,9 @@ pub trait SubsystemContext: Send + 'static { fn send_unbounded_message(&mut self, msg: X) where Self::OutgoingMessages: From + Send, - Self::Sender: SubsystemSender, X: Send, { - // self.sender().send_unbounded_message(::from(msg)) + self.sender().send_unbounded_message(::from(msg)) } /// Obtain the sender. From cf8f65dd7527b24e3503bdcce2024e9d48022a1c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 16:41:06 +0200 Subject: [PATCH 077/139] remove `no_dispatch` from parser --- node/overseer/overseer-gen/examples/duo.rs | 4 +-- node/overseer/overseer-gen/examples/misc.rs | 24 +------------ node/overseer/overseer-gen/examples/solo.rs | 4 +-- .../src/parse/parse_overseer_struct.rs | 25 +++----------- .../proc-macro/src/parse/tests.rs | 19 +++-------- .../overseer-gen/proc-macro/src/tests.rs | 8 ++--- .../tests/ui/err-04-missing-error.rs | 2 +- .../tests/ui/err-05-missing-field.rs | 4 +-- .../tests/ui/err-06-missing-subsystem.rs | 2 +- .../tests/ui/err-07-missing-spawner.rs | 2 +- .../tests/ui/err-08-duplicate-subsystem.rs | 2 +- .../tests/ui/err-09-uninit_generic_baggage.rs | 2 +- node/overseer/src/lib.rs | 34 +++++++++---------- 13 files changed, 42 insertions(+), 90 deletions(-) diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs index 3800bc08395c..f91d1b5cc88e 100644 --- a/node/overseer/overseer-gen/examples/duo.rs +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -44,12 +44,12 @@ impl GoblinTower { } } -#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] +#[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)] struct Duo { #[subsystem(consumes: MsgStrukt, sends: [Plinko])] sub0: AwesomeSubSys, - #[subsystem(no_dispatch, blocking, consumes: Plinko, sends: [MsgStrukt])] + #[subsystem(blocking, consumes: Plinko, sends: [MsgStrukt])] plinkos: GoblinTower, i_like_pi: f64, diff --git a/node/overseer/overseer-gen/examples/misc.rs b/node/overseer/overseer-gen/examples/misc.rs index 19b85f3a2f8c..cfbb56d04005 100644 --- a/node/overseer/overseer-gen/examples/misc.rs +++ b/node/overseer/overseer-gen/examples/misc.rs @@ -33,7 +33,7 @@ pub struct EvX; impl EvX { pub fn focus<'a, T>(&'a self) -> Result { - unimplemented!("dispatch") + unimplemented!("focus") } } @@ -65,25 +65,3 @@ pub struct MsgStrukt(pub u8); #[derive(Debug, Clone, Copy)] pub struct Plinko; - -impl From for MsgStrukt { - fn from(_event: NetworkMsg) -> Self { - MsgStrukt(1u8) - } -} - -#[derive(Debug, Clone, Copy)] -pub enum NetworkMsg { - A, - B, - C, -} - -impl NetworkMsg { - pub fn focus(&self) -> Result { - Ok(match self { - Self::B => return Err(WrongVariant), - Self::A | Self::C => self.clone(), - }) - } -} diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index 9997531fa798..e5e9cc1a6d13 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -6,9 +6,9 @@ mod misc; pub use self::misc::*; -#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] +#[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)] struct Solo { - #[subsystem(no_dispatch, consumes: Plinko, sends: [MsgStrukt])] + #[subsystem(consumes: Plinko, sends: [MsgStrukt])] goblin_tower: GoblinTower, } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs index 36c60cd84f60..d51998b4ce6c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs @@ -30,7 +30,6 @@ use quote::{quote, ToTokens}; mod kw { syn::custom_keyword!(wip); - syn::custom_keyword!(no_dispatch); syn::custom_keyword!(blocking); syn::custom_keyword!(consumes); syn::custom_keyword!(sends); @@ -44,9 +43,6 @@ pub(crate) enum SubSysAttrItem { /// The subsystem is blocking and requires to be /// spawned on an exclusive thread. Blocking(kw::blocking), - /// External messages should not be - after being converted - - /// be dispatched to the annotated subsystem. - NoDispatch(kw::no_dispatch), /// Message to be sent by this subsystem. Sends(Sends), /// Message to be consumed by this subsystem. @@ -60,8 +56,6 @@ impl Parse for SubSysAttrItem { Self::Wip(input.parse::()?) } else if lookahead.peek(kw::blocking) { Self::Blocking(input.parse::()?) - } else if lookahead.peek(kw::no_dispatch) { - Self::NoDispatch(input.parse::()?) } else if lookahead.peek(kw::sends) { Self::Sends(input.parse::()?) } else { @@ -79,9 +73,6 @@ impl ToTokens for SubSysAttrItem { Self::Blocking(blocking) => { quote! { #blocking } }, - Self::NoDispatch(no_dispatch) => { - quote! { #no_dispatch } - }, Self::Sends(_) => { quote! {} }, @@ -94,7 +85,7 @@ impl ToTokens for SubSysAttrItem { } /// A field of the struct annotated with -/// `#[subsystem(no_dispatch, , A | B | C)]` +/// `#[subsystem(A, B, C)]` #[derive(Clone, Debug)] pub(crate) struct SubSysField { /// Name of the field. @@ -107,9 +98,6 @@ pub(crate) struct SubSysField { pub(crate) message_to_consume: Path, /// Types of messages to be sent by the subsystem. pub(crate) messages_to_send: Vec, - /// If `no_dispatch` is present, if the message is incoming via - /// an `extern` `Event`, it will not be dispatched to all subsystems. - pub(crate) no_dispatch: bool, /// If the subsystem implementation is blocking execution and hence /// has to be spawned on a separate thread or thread pool. pub(crate) blocking: bool, @@ -206,12 +194,10 @@ impl Parse for Consumes { } } -/// Parses `(no_dispatch, Foo, sends = [Bar, Baz])` +/// Parses `(Foo, sends = [Bar, Baz])` /// including the `(` and `)`. #[derive(Debug, Clone)] pub(crate) struct SubSystemAttrItems { - #[allow(dead_code)] - pub(crate) no_dispatch: bool, /// The subsystem is in progress, only generate the `Wrapper` variant, but do not forward messages /// and also not include the subsystem in the list of subsystems. pub(crate) wip: bool, @@ -258,11 +244,10 @@ impl Parse for SubSystemAttrItems { )) } - let no_dispatch = extract_variant!(unique, NoDispatch; default = false); let blocking = extract_variant!(unique, Blocking; default = false); let wip = extract_variant!(unique, Wip; default = false); - Ok(Self { no_dispatch, blocking, wip, sends, consumes }) + Ok(Self { blocking, wip, sends, consumes }) } } @@ -479,8 +464,7 @@ impl OverseerGuts { } unique_subsystem_idents.insert(generic.clone()); - let SubSystemAttrItems { no_dispatch, wip, blocking, consumes, sends, .. } = - subsystem_attrs; + let SubSystemAttrItems { wip, blocking, consumes, sends, .. } = subsystem_attrs; // messages to be sent let sends = if let Some(sends) = sends { @@ -501,7 +485,6 @@ impl OverseerGuts { generic, message_to_consume: consumes, messages_to_send: sends, - no_dispatch, wip, blocking, }); diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs index b188b451ba80..990b5d6872c0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/tests.rs @@ -67,15 +67,6 @@ mod strukt { }); } - #[test] - fn parse_subsystem_attr_item_works_01_no_dispatch() { - assert_matches!( - syn::parse2::(quote! { - no_dispatch - }), Ok(SubSysAttrItem::NoDispatch(_)) => { - }); - } - #[test] fn parse_subsystem_attr_item_works_02_sends() { assert_matches!( @@ -137,7 +128,7 @@ mod strukt { #[test] fn parse_subsystem_attributes_works_00() { syn::parse2::(quote! { - (wip, no_dispatch, blocking, consumes: Foo, sends: []) + (wip, blocking, consumes: Foo, sends: []) }) .unwrap(); } @@ -219,7 +210,7 @@ mod strukt { fn parse_subsystem_attributes_works_09_neither_consumes_nor_sends() { assert_matches!( syn::parse2::(quote! { - (no_dispatch, sends: []) + (sends: []) }), Err(e) => { // must either consume smth or sends smth, neither is NOK dbg!(e) @@ -260,7 +251,7 @@ mod strukt { fn struct_parse_baggage() { let item: OverseerGuts = parse_quote! { pub struct Ooooh where X: Secrit { - #[subsystem(no_dispatch, consumes: Foo, sends: [])] + #[subsystem(consumes: Foo, sends: [])] sub0: FooSubsystem, metrics: Metrics, @@ -273,13 +264,13 @@ mod strukt { fn struct_parse_full() { let item: OverseerGuts = parse_quote! { pub struct Ooooh where X: Secrit { - #[subsystem(no_dispatch, consumes: Foo, sends: [])] + #[subsystem(consumes: Foo, sends: [])] sub0: FooSubsystem, #[subsystem(blocking, consumes: Bar, sends: [])] yyy: BaersBuyBilliardBalls, - #[subsystem(no_dispatch, blocking, consumes: Twain, sends: [])] + #[subsystem(blocking, consumes: Twain, sends: [])] fff: Beeeeep, #[subsystem(consumes: Rope)] diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 89803f3761a4..364e53f5b02a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -32,13 +32,13 @@ fn print() { let item = quote! { pub struct Ooooh where X: Secrit { - #[subsystem(no_dispatch, Foo)] + #[subsystem(Foo)] sub0: FooSubsystem, #[subsystem(blocking, Bar)] yyy: BaersBuyBilliardBalls, - #[subsystem(no_dispatch, blocking, Twain)] + #[subsystem(blocking, Twain)] fff: Beeeeep, #[subsystem(Rope)] @@ -57,13 +57,13 @@ fn print() { fn struct_parse_full() { let item: OverseerGuts = parse_quote! { pub struct Ooooh where X: Secrit { - #[subsystem(no_dispatch, Foo)] + #[subsystem(Foo)] sub0: FooSubsystem, #[subsystem(blocking, Bar)] yyy: BaersBuyBilliardBalls, - #[subsystem(no_dispatch, blocking, Twain)] + #[subsystem(blocking, Twain)] fff: Beeeeep, #[subsystem(Rope)] diff --git a/node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs index 3b6966f1da79..3547eb36a5c5 100644 --- a/node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs +++ b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs @@ -15,7 +15,7 @@ struct MsgStrukt(u8); #[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(no_dispatch, MsgStrukt)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, i_like_pie: f64, diff --git a/node/overseer/overseer-gen/tests/ui/err-05-missing-field.rs b/node/overseer/overseer-gen/tests/ui/err-05-missing-field.rs index a51248b5406a..22ead71ea180 100644 --- a/node/overseer/overseer-gen/tests/ui/err-05-missing-field.rs +++ b/node/overseer/overseer-gen/tests/ui/err-05-missing-field.rs @@ -21,7 +21,7 @@ pub struct MsgStrukt(u8); #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(no_dispatch, MsgStrukt)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, i_like_pie: f64, } @@ -58,4 +58,4 @@ fn main() { .spawner(DummySpawner) .build() .unwrap(); -} \ No newline at end of file +} diff --git a/node/overseer/overseer-gen/tests/ui/err-06-missing-subsystem.rs b/node/overseer/overseer-gen/tests/ui/err-06-missing-subsystem.rs index 540ed366a054..bccc990d5e66 100644 --- a/node/overseer/overseer-gen/tests/ui/err-06-missing-subsystem.rs +++ b/node/overseer/overseer-gen/tests/ui/err-06-missing-subsystem.rs @@ -21,7 +21,7 @@ pub struct MsgStrukt(u8); #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(no_dispatch, MsgStrukt)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, i_like_pie: f64, } diff --git a/node/overseer/overseer-gen/tests/ui/err-07-missing-spawner.rs b/node/overseer/overseer-gen/tests/ui/err-07-missing-spawner.rs index bc25e0c47cff..4193d28ec879 100644 --- a/node/overseer/overseer-gen/tests/ui/err-07-missing-spawner.rs +++ b/node/overseer/overseer-gen/tests/ui/err-07-missing-spawner.rs @@ -21,7 +21,7 @@ pub struct MsgStrukt(u8); #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(no_dispatch, MsgStrukt)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, i_like_pie: f64, } diff --git a/node/overseer/overseer-gen/tests/ui/err-08-duplicate-subsystem.rs b/node/overseer/overseer-gen/tests/ui/err-08-duplicate-subsystem.rs index ea2ad0b4d1ac..8895723abbc1 100644 --- a/node/overseer/overseer-gen/tests/ui/err-08-duplicate-subsystem.rs +++ b/node/overseer/overseer-gen/tests/ui/err-08-duplicate-subsystem.rs @@ -21,7 +21,7 @@ pub struct MsgStrukt(u8); #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(no_dispatch, MsgStrukt)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, i_like_pie: f64, } diff --git a/node/overseer/overseer-gen/tests/ui/err-09-uninit_generic_baggage.rs b/node/overseer/overseer-gen/tests/ui/err-09-uninit_generic_baggage.rs index 06ce22dba351..64a2f6c4bffc 100644 --- a/node/overseer/overseer-gen/tests/ui/err-09-uninit_generic_baggage.rs +++ b/node/overseer/overseer-gen/tests/ui/err-09-uninit_generic_baggage.rs @@ -21,7 +21,7 @@ pub struct MsgStrukt(u8); #[overlord(signal=SigSigSig, error=OverseerError, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(no_dispatch, MsgStrukt)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, i_like_pie: T, } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 0d48539e8d1b..321ce906bdb2 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -418,18 +418,18 @@ pub async fn forward_events>(client: Arc

, mut hand message_capacity=2048, )] pub struct Overseer { - #[subsystem(no_dispatch, CandidateValidationMessage, sends: [ + #[subsystem(CandidateValidationMessage, sends: [ RuntimeApiMessage, ])] candidate_validation: CandidateValidation, - #[subsystem(no_dispatch, PvfCheckerMessage, sends: [ + #[subsystem(PvfCheckerMessage, sends: [ CandidateValidationMessage, RuntimeApiMessage, ])] pvf_checker: PvfChecker, - #[subsystem(no_dispatch, CandidateBackingMessage, sends: [ + #[subsystem(CandidateBackingMessage, sends: [ CandidateValidationMessage, CollatorProtocolMessage, AvailabilityDistributionMessage, @@ -448,7 +448,7 @@ pub struct Overseer { ])] statement_distribution: StatementDistribution, - #[subsystem(no_dispatch, AvailabilityDistributionMessage, sends: [ + #[subsystem(AvailabilityDistributionMessage, sends: [ AvailabilityStoreMessage, AvailabilityRecoveryMessage, ChainApiMessage, @@ -457,14 +457,14 @@ pub struct Overseer { ])] availability_distribution: AvailabilityDistribution, - #[subsystem(no_dispatch, AvailabilityRecoveryMessage, sends: [ + #[subsystem(AvailabilityRecoveryMessage, sends: [ NetworkBridgeMessage, RuntimeApiMessage, AvailabilityStoreMessage, ])] availability_recovery: AvailabilityRecovery, - #[subsystem(blocking, no_dispatch, BitfieldSigningMessage, sends: [ + #[subsystem(blocking, BitfieldSigningMessage, sends: [ AvailabilityStoreMessage, RuntimeApiMessage, BitfieldDistributionMessage, @@ -478,7 +478,7 @@ pub struct Overseer { ])] bitfield_distribution: BitfieldDistribution, - #[subsystem(no_dispatch, ProvisionerMessage, sends: [ + #[subsystem(ProvisionerMessage, sends: [ RuntimeApiMessage, CandidateBackingMessage, ChainApiMessage, @@ -486,16 +486,16 @@ pub struct Overseer { ])] provisioner: Provisioner, - #[subsystem(no_dispatch, blocking, RuntimeApiMessage, sends: [])] + #[subsystem(blocking, RuntimeApiMessage, sends: [])] runtime_api: RuntimeApi, - #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage, sends: [ + #[subsystem(blocking, AvailabilityStoreMessage, sends: [ ChainApiMessage, RuntimeApiMessage, ])] availability_store: AvailabilityStore, - #[subsystem(no_dispatch, NetworkBridgeMessage, sends: [ + #[subsystem(NetworkBridgeMessage, sends: [ ApprovalDistributionMessage, StatementDistributionMessage, BitfieldDistributionMessage, @@ -506,16 +506,16 @@ pub struct Overseer { ])] network_bridge: NetworkBridge, - #[subsystem(no_dispatch, blocking, ChainApiMessage, sends: [])] + #[subsystem(blocking, ChainApiMessage, sends: [])] chain_api: ChainApi, - #[subsystem(no_dispatch, CollationGenerationMessage, sends: [ + #[subsystem(CollationGenerationMessage, sends: [ RuntimeApiMessage, CollatorProtocolMessage, ])] collation_generation: CollationGeneration, - #[subsystem(no_dispatch, CollatorProtocolMessage, sends: [ + #[subsystem(CollatorProtocolMessage, sends: [ NetworkBridgeMessage, RuntimeApiMessage, CandidateBackingMessage, @@ -528,7 +528,7 @@ pub struct Overseer { ])] approval_distribution: ApprovalDistribution, - #[subsystem(no_dispatch, blocking, ApprovalVotingMessage, sends: [ + #[subsystem(blocking, ApprovalVotingMessage, sends: [ RuntimeApiMessage, ChainApiMessage, ChainSelectionMessage, @@ -546,7 +546,7 @@ pub struct Overseer { ])] gossip_support: GossipSupport, - #[subsystem(no_dispatch, blocking, DisputeCoordinatorMessage, sends: [ + #[subsystem(blocking, DisputeCoordinatorMessage, sends: [ RuntimeApiMessage, ChainApiMessage, DisputeDistributionMessage, @@ -556,14 +556,14 @@ pub struct Overseer { ])] dispute_coordinator: DisputeCoordinator, - #[subsystem(no_dispatch, DisputeDistributionMessage, sends: [ + #[subsystem(DisputeDistributionMessage, sends: [ RuntimeApiMessage, DisputeCoordinatorMessage, NetworkBridgeMessage, ])] dispute_distribution: DisputeDistribution, - #[subsystem(no_dispatch, blocking, ChainSelectionMessage, sends: [ChainApiMessage])] + #[subsystem(blocking, ChainSelectionMessage, sends: [ChainApiMessage])] chain_selection: ChainSelection, /// External listeners waiting for a hash to be in the active-leave set. From 40aab170af1142d00edc21e07c7feea9c0d85f40 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 17:04:57 +0200 Subject: [PATCH 078/139] remove dead extern_network_ty, it's unused --- node/overseer/overseer-gen/proc-macro/src/lib.rs | 1 - .../overseer-gen/proc-macro/src/parse/mod.rs | 1 - .../proc-macro/src/parse/parse_overseer_attr.rs | 16 ---------------- .../src/parse/parse_overseer_struct.rs | 3 --- node/overseer/src/lib.rs | 1 - 5 files changed, 22 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index b4eb2273a41b..92aafcf62ee5 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -92,7 +92,6 @@ pub(crate) fn impl_overseer_gen( extern_event_ty: args.extern_event_ty, extern_signal_ty: args.extern_signal_ty, extern_error_ty: args.extern_error_ty, - extern_network_ty: args.extern_network_ty, outgoing_ty: args.outgoing_ty, }; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs index 42ef44a02336..518072d459e0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs @@ -18,7 +18,6 @@ mod kw { syn::custom_keyword!(event); syn::custom_keyword!(signal); syn::custom_keyword!(error); - syn::custom_keyword!(network); syn::custom_keyword!(outgoing); syn::custom_keyword!(gen); syn::custom_keyword!(signal_capacity); diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs index 9bd9a5ab6212..633a2cef4e24 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs @@ -28,7 +28,6 @@ use syn::{ #[derive(Clone, Debug)] enum OverseerAttrItem { ExternEventType { tag: kw::event, eq_token: Token![=], value: Path }, - ExternNetworkType { tag: kw::network, eq_token: Token![=], value: Path }, ExternOverseerSignalType { tag: kw::signal, eq_token: Token![=], value: Path }, ExternErrorType { tag: kw::error, eq_token: Token![=], value: Path }, OutgoingType { tag: kw::outgoing, eq_token: Token![=], value: Path }, @@ -43,9 +42,6 @@ impl ToTokens for OverseerAttrItem { Self::ExternEventType { tag, eq_token, value } => { quote! { #tag #eq_token, #value } }, - Self::ExternNetworkType { tag, eq_token, value } => { - quote! { #tag #eq_token, #value } - }, Self::ExternOverseerSignalType { tag, eq_token, value } => { quote! { #tag #eq_token, #value } }, @@ -90,12 +86,6 @@ impl Parse for OverseerAttrItem { eq_token: input.parse()?, value: input.parse()?, }) - } else if lookahead.peek(kw::network) { - Ok(OverseerAttrItem::ExternNetworkType { - tag: input.parse::()?, - eq_token: input.parse()?, - value: input.parse()?, - }) } else if lookahead.peek(kw::outgoing) { Ok(OverseerAttrItem::OutgoingType { tag: input.parse::()?, @@ -133,10 +123,6 @@ pub(crate) struct OverseerAttrArgs { pub(crate) extern_event_ty: Path, pub(crate) extern_signal_ty: Path, pub(crate) extern_error_ty: Path, - /// A external subsystem that both consumes and produces messages - /// but is not part of the band of subsystems, it's a mere proxy - /// to another entity that consumes/produces messages. - pub(crate) extern_network_ty: Option, pub(crate) outgoing_ty: Option, pub(crate) signal_channel_capacity: usize, pub(crate) message_channel_capacity: usize, @@ -190,7 +176,6 @@ impl Parse for OverseerAttrArgs { let event = extract_variant!(unique, ExternEventType; err = "Must declare the overseer event type via `event=..`.")?; let signal = extract_variant!(unique, ExternOverseerSignalType; err = "Must declare the overseer signal type via `span=..`.")?; let message_wrapper = extract_variant!(unique, MessageWrapperName; err = "Must declare the overseer generated wrapping message type via `gen=..`.")?; - let network = extract_variant!(unique, ExternNetworkType); let outgoing = extract_variant!(unique, OutgoingType); Ok(OverseerAttrArgs { @@ -199,7 +184,6 @@ impl Parse for OverseerAttrArgs { extern_event_ty: event, extern_signal_ty: signal, extern_error_ty: error, - extern_network_ty: network, outgoing_ty: outgoing, message_wrapper, }) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs index d51998b4ce6c..d683c303371d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs @@ -287,9 +287,6 @@ pub(crate) struct OverseerInfo { /// Incoming event type from the outer world, usually an external framework of some sort. pub(crate) extern_event_ty: Path, - /// Incoming event type from an external entity, commonly from the network. - pub(crate) extern_network_ty: Option, - /// Type of messages that are sent to an external subsystem. /// Merely here to be included during generation of `#message_wrapper` type. pub(crate) outgoing_ty: Option, diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 321ce906bdb2..7f610ab7c2b4 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -414,7 +414,6 @@ pub async fn forward_events>(client: Arc

, mut hand event=Event, signal=OverseerSignal, error=SubsystemError, - network=NetworkBridgeEvent, message_capacity=2048, )] pub struct Overseer { From 422e155837c658daee13da64a3139573a9d817ab Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 29 Apr 2022 17:18:39 +0200 Subject: [PATCH 079/139] overseer test --- node/overseer/overseer-gen/examples/duo.rs | 1 - node/overseer/overseer-gen/examples/solo.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs index f91d1b5cc88e..ffb6c0cd8d5a 100644 --- a/node/overseer/overseer-gen/examples/duo.rs +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -57,7 +57,6 @@ struct Duo { i_like_hash: HashMap, } -#[test] fn main() { let (overseer, _handle): (Duo<_, f64>, _) = Duo::builder() .sub0(AwesomeSubSys::default()) diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index e5e9cc1a6d13..f3b718bada6c 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -30,7 +30,6 @@ impl GoblinTower { } } -#[test] fn main() { let (overseer, _handle): (Solo<_>, _) = Solo::builder() .goblin_tower(GoblinTower::default()) From 7e8ecf6c5779d33203b20d4088ec7ffdb2b6878f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 2 May 2022 11:30:02 +0200 Subject: [PATCH 080/139] chore: make test cases for overseer more realistic Name in struct Overseer does not necessarily match subsystem name. --- node/overseer/overseer-gen/examples/duo.rs | 12 ++++++------ node/overseer/overseer-gen/examples/solo.rs | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs index ffb6c0cd8d5a..b50c671eddf0 100644 --- a/node/overseer/overseer-gen/examples/duo.rs +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -11,7 +11,7 @@ pub use self::misc::*; #[derive(Default)] pub struct AwesomeSubSys; -#[overseer::subsystem(error=Yikes)] +#[overseer::subsystem(Awesome, error=Yikes)] impl AwesomeSubSys { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); @@ -27,10 +27,10 @@ impl AwesomeSubSys { } #[derive(Default)] -pub struct GoblinTower; +pub struct Fortified; -#[overseer::subsystem(error=Yikes)] -impl GoblinTower { +#[overseer::subsystem(GoblinTower, error=Yikes)] +impl Fortified { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( @@ -47,7 +47,7 @@ impl GoblinTower { #[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)] struct Duo { #[subsystem(consumes: MsgStrukt, sends: [Plinko])] - sub0: AwesomeSubSys, + sub0: Awesome, #[subsystem(blocking, consumes: Plinko, sends: [MsgStrukt])] plinkos: GoblinTower, @@ -60,7 +60,7 @@ struct Duo { fn main() { let (overseer, _handle): (Duo<_, f64>, _) = Duo::builder() .sub0(AwesomeSubSys::default()) - .plinkos(GoblinTower::default()) + .plinkos(Fortified::default()) .i_like_pi(::std::f64::consts::PI) .i_like_generic(42.0) .i_like_hash(HashMap::new()) diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index f3b718bada6c..9d0a6fefe839 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -13,10 +13,10 @@ struct Solo { } #[derive(Default)] -pub struct GoblinTower; +pub struct Fortified; -#[overseer::subsystem(error=Yikes)] -impl GoblinTower { +#[overseer::subsystem(GoblinTower, error=Yikes)] +impl Fortified { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let mut sender = ctx.sender().clone(); ctx.spawn( @@ -32,7 +32,7 @@ impl GoblinTower { fn main() { let (overseer, _handle): (Solo<_>, _) = Solo::builder() - .goblin_tower(GoblinTower::default()) + .goblin_tower(Fortified::default()) .spawner(DummySpawner) .build() .unwrap(); From 01f4fffc2cb056dc405589b70a18d5d19ffc4db8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 2 May 2022 11:30:34 +0200 Subject: [PATCH 081/139] fix misgeneration, `\#outgoing_wrapper` is not be prefixed with `\#support_crate` --- .../overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index b3e2889beb32..d7731be8dced 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -409,7 +409,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, #all_messages_wrapper: From< #outgoing_wrapper >, #all_messages_wrapper: From< #consumes >, - #support_crate :: #outgoing_wrapper: #( From< #outgoing > )+*, + #outgoing_wrapper: #( From< #outgoing > )+*, }; From c601dcc9dbb0026c55913d5b833b4cc2704a5b38 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 2 May 2022 11:31:52 +0200 Subject: [PATCH 082/139] make overseer stuff prep --- node/collation-generation/src/lib.rs | 2 +- node/core/av-store/src/lib.rs | 2 +- node/core/candidate-validation/src/lib.rs | 2 +- node/core/dispute-coordinator/src/lib.rs | 2 +- node/network/availability-distribution/src/lib.rs | 2 +- node/network/availability-recovery/src/lib.rs | 2 +- node/network/collator-protocol/src/lib.rs | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index ea0b459eb281..74be0548888c 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -164,7 +164,7 @@ impl CollationGenerationSubsystem { } } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(CollationGeneration, error=SubsystemError)] impl CollationGenerationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index f1f2b5d432dc..1c4fb859710c 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -519,7 +519,7 @@ impl KnownUnfinalizedBlocks { } } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(AvailabilityStore, error=SubsystemError)] impl AvailabilityStoreSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(self, ctx).map(|_| Ok(())).boxed(); diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 252657046eb0..c87cc1b9cc9b 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -93,7 +93,7 @@ impl CandidateValidationSubsystem { } } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(CandidateValidation, error=SubsystemError)] impl CandidateValidationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run( diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 21ae0cbcf002..f1a36c3305e0 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -124,7 +124,7 @@ impl Config { } } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(DisputeCoordinator, error=SubsystemError)] impl DisputeCoordinatorSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async { diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index f3c9003c175f..bc546b18df04 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -68,7 +68,7 @@ pub struct IncomingRequestReceivers { pub chunk_req_receiver: IncomingRequestReceiver, } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(AvailabilityDistribution, error=SubsystemError)] impl AvailabilityDistributionSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index e15ecf6b5399..e19557304d7e 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -806,7 +806,7 @@ impl Default for State { } } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(AvailabilityRecovery, error=SubsystemError)] impl AvailabilityRecoverySubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index b748c1531f6b..75e6e903097e 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -113,7 +113,7 @@ impl CollatorProtocolSubsystem { } } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(CollatorProtocol, error=SubsystemError)] impl CollatorProtocolSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self From d5241f79bba0ca4ef8e624a6ca4faa2d2e4ee4b6 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 2 May 2022 11:32:03 +0200 Subject: [PATCH 083/139] chore fmt --- node/network/bridge/src/lib.rs | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 5689a8c76b25..2faf76181b4f 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -1063,15 +1063,15 @@ fn dispatch_validation_event_to_all_unbounded( ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { match event { - NetworkBridgeEvent::::BitfieldDistribution(bdm) => { - ctx.send_unbounded_message(bdm) - } - NetworkBridgeEvent::::StatementDistribution(sdm) => { - ctx.send_unbounded_message(sdm) - } - NetworkBridgeEvent::::ApprovalDistribution(adm) => { - ctx.send_unbounded_message(adm) - } + NetworkBridgeEvent::::BitfieldDistribution( + bdm, + ) => ctx.send_unbounded_message(bdm), + NetworkBridgeEvent::::StatementDistribution( + sdm, + ) => ctx.send_unbounded_message(sdm), + NetworkBridgeEvent::::ApprovalDistribution( + adm, + ) => ctx.send_unbounded_message(adm), } } @@ -1079,21 +1079,19 @@ fn dispatch_collation_event_to_all_unbounded( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeSenderTrait, ) { - match event { - NetworkBridgeEvent::::BitfieldDistribution(bdm) => { - ctx.send_unbounded_message(bdm) - } - NetworkBridgeEvent::::StatementDistribution(sdm) => { - ctx.send_unbounded_message(sdm) - } - NetworkBridgeEvent::::ApprovalDistribution(adm) => { - ctx.send_unbounded_message(adm) - } + NetworkBridgeEvent::::BitfieldDistribution( + bdm, + ) => ctx.send_unbounded_message(bdm), + NetworkBridgeEvent::::StatementDistribution( + sdm, + ) => ctx.send_unbounded_message(sdm), + NetworkBridgeEvent::::ApprovalDistribution( + adm, + ) => ctx.send_unbounded_message(adm), } } - async fn dispatch_validation_events_to_all( events: I, ctx: &mut impl overseer::NetworkBridgeSenderTrait, @@ -1105,7 +1103,9 @@ async fn dispatch_validation_events_to_all( sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; - sender.send_messages(event.focus().map(StatementDistributionMessage::from)).await; + sender + .send_messages(event.focus().map(StatementDistributionMessage::from)) + .await; } } From 8132e173420741ccd2caa3ab52318a6f2c41063e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 2 May 2022 11:32:26 +0200 Subject: [PATCH 084/139] make it all work together `polkadot-overseer` is compiling! --- .../overseer-gen/proc-macro/src/lib.rs | 137 +++--------------- .../overseer-gen/proc-macro/src/overseer.rs | 65 +++++++++ .../overseer-gen/proc-macro/src/parse/mod.rs | 2 + .../src/parse/parse_overseer_struct.rs | 4 +- .../src/parse/parse_subsystem_attr.rs | 40 +++-- .../overseer-gen/proc-macro/src/subsystem.rs | 72 +++++++++ 6 files changed, 192 insertions(+), 128 deletions(-) create mode 100644 node/overseer/overseer-gen/proc-macro/src/overseer.rs create mode 100644 node/overseer/overseer-gen/proc-macro/src/subsystem.rs diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 92aafcf62ee5..2d7347bce675 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -25,7 +25,12 @@ mod impl_channels_out; mod impl_message_wrapper; mod impl_overseer; mod impl_subsystem_ctx_sender; +mod overseer; mod parse; +mod subsystem; + +#[cfg(test)] +mod tests; use impl_builder::*; use impl_channels_out::*; @@ -34,8 +39,22 @@ use impl_overseer::*; use impl_subsystem_ctx_sender::*; use parse::*; -#[cfg(test)] -mod tests; +use self::{overseer::*, subsystem::*}; + +/// Obtain the support crate `Path` as `TokenStream`. +pub(crate) fn support_crate() -> TokenStream { + if cfg!(test) { + quote! {crate} + } else { + use proc_macro_crate::{crate_name, FoundCrate}; + let crate_name = crate_name("polkadot-overseer-gen") + .expect("Support crate `polkadot-overseer-gen` is present in `Cargo.toml`. qed"); + match crate_name { + FoundCrate::Itself => quote! {crate}, + FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(), + } + } +} #[proc_macro_attribute] pub fn overlord( @@ -60,117 +79,3 @@ pub fn subsystem( .unwrap_or_else(|err| err.to_compile_error()) .into() } - -pub(crate) fn impl_overseer_gen( - attr: TokenStream, - orig: TokenStream, -) -> Result { - let args: OverseerAttrArgs = parse2(attr)?; - let message_wrapper = args.message_wrapper; - - let of: OverseerGuts = parse2(orig)?; - - let support_crate_name = if cfg!(test) { - quote! {crate} - } else { - use proc_macro_crate::{crate_name, FoundCrate}; - let crate_name = crate_name("polkadot-overseer-gen") - .expect("Support crate polkadot-overseer-gen is present in `Cargo.toml`. qed"); - match crate_name { - FoundCrate::Itself => quote! {crate}, - FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(), - } - }; - let info = OverseerInfo { - support_crate_name, - subsystems: of.subsystems, - baggage: of.baggage, - overseer_name: of.name, - message_wrapper, - message_channel_capacity: args.message_channel_capacity, - signal_channel_capacity: args.signal_channel_capacity, - extern_event_ty: args.extern_event_ty, - extern_signal_ty: args.extern_signal_ty, - extern_error_ty: args.extern_error_ty, - outgoing_ty: args.outgoing_ty, - }; - - let mut additive = impl_overseer_struct(&info); - additive.extend(impl_builder(&info)); - - additive.extend(impl_overseen_subsystem(&info)); - additive.extend(impl_channels_out_struct(&info)); - additive.extend(impl_subsystem(&info)?); - - additive.extend(impl_message_wrapper_enum(&info)?); - - let ts = expander::Expander::new("overlord-expansion") - .add_comment("Generated overseer code by `#[overlord(..)]`".to_owned()) - .dry(!cfg!(feature = "expand")) - .verbose(true) - .fmt(expander::Edition::_2021) - .write_to_out_dir(additive) - .expect("Expander does not fail due to IO in OUT_DIR. qed"); - - Ok(ts) -} - -pub(crate) fn impl_subsystem_gen( - attr: TokenStream, - orig: TokenStream, -) -> Result { - let error_ty = parse2::(attr.clone())?.extern_error_ty; - - let mut struktured_impl: syn::ItemImpl = parse2(orig)?; - - let support_crate = if cfg!(test) { - quote! {crate} - } else { - use proc_macro_crate::{crate_name, FoundCrate}; - let crate_name = crate_name("polkadot-overseer-gen") - .expect("Support crate polkadot-overseer-gen is present in `Cargo.toml`. qed"); - match crate_name { - FoundCrate::Itself => quote! {crate}, - FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(), - } - }; - - let me = match &*struktured_impl.self_ty { - syn::Type::Path(ref path) => dbg!(path) - .path - .segments - .last() - .expect("Must be ident. FIXME bail properly") - .ident - .clone(), - _ => return Err(syn::Error::new(attr.span(), "Doesn't take any arguments at this time")), - }; - let subsystem_ctx_trait = format_ident!("{}ContextTrait", me); - let subsystem_sender_trait = format_ident!("{}SenderTrait", me); - let extra_where_predicates: Punctuated = parse_quote! { - Context: #subsystem_ctx_trait, - Context: #support_crate::SubsystemContext, - ::Sender: #subsystem_sender_trait, - ::Sender: #subsystem_sender_trait, - }; - struktured_impl.trait_.replace(( - None, - parse_quote! { - #support_crate::Subsystem - }, - syn::token::For::default(), - )); - let where_clause = struktured_impl.generics.make_where_clause(); - where_clause.predicates.extend(extra_where_predicates); - - let ts = - expander::Expander::new(format!("subsystem-{}-expansion", me.to_string().to_lowercase())) - .add_comment("Generated overseer code by `#[subsystem(..)]`".to_owned()) - .dry(!cfg!(feature = "expand")) - .verbose(true) - .fmt(expander::Edition::_2021) - .write_to_out_dir(struktured_impl.into_token_stream()) - .expect("Expander does not fail due to IO in OUT_DIR. qed"); - - Ok(ts) -} diff --git a/node/overseer/overseer-gen/proc-macro/src/overseer.rs b/node/overseer/overseer-gen/proc-macro/src/overseer.rs new file mode 100644 index 000000000000..b3aee62428c1 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/overseer.rs @@ -0,0 +1,65 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use proc_macro2::{Ident, Span, TokenStream}; +use quote::{format_ident, quote, ToTokens}; +use syn::{parse2, parse_quote, punctuated::Punctuated, spanned::Spanned, Result}; + +use super::{parse::*, *}; + +pub(crate) fn impl_overseer_gen( + attr: TokenStream, + orig: TokenStream, +) -> Result { + let args: OverseerAttrArgs = parse2(attr)?; + let message_wrapper = args.message_wrapper; + + let of: OverseerGuts = parse2(orig)?; + + let support_crate = support_crate(); + let info = OverseerInfo { + support_crate, + subsystems: of.subsystems, + baggage: of.baggage, + overseer_name: of.name, + message_wrapper, + message_channel_capacity: args.message_channel_capacity, + signal_channel_capacity: args.signal_channel_capacity, + extern_event_ty: args.extern_event_ty, + extern_signal_ty: args.extern_signal_ty, + extern_error_ty: args.extern_error_ty, + outgoing_ty: args.outgoing_ty, + }; + + let mut additive = impl_overseer_struct(&info); + additive.extend(impl_builder(&info)); + + additive.extend(impl_overseen_subsystem(&info)); + additive.extend(impl_channels_out_struct(&info)); + additive.extend(impl_subsystem(&info)?); + + additive.extend(impl_message_wrapper_enum(&info)?); + + let ts = expander::Expander::new("overlord-expansion") + .add_comment("Generated overseer code by `#[overlord(..)]`".to_owned()) + .dry(!cfg!(feature = "expand")) + .verbose(true) + .fmt(expander::Edition::_2021) + .write_to_out_dir(additive) + .expect("Expander does not fail due to IO in OUT_DIR. qed"); + + Ok(ts) +} diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs index 518072d459e0..e5166464ffdf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . mod kw { + /// An external event. syn::custom_keyword!(event); syn::custom_keyword!(signal); syn::custom_keyword!(error); @@ -22,6 +23,7 @@ mod kw { syn::custom_keyword!(gen); syn::custom_keyword!(signal_capacity); syn::custom_keyword!(message_capacity); + syn::custom_keyword!(subsystem); } mod parse_overseer_attr; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs index d683c303371d..317b47d129ea 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs @@ -263,7 +263,7 @@ pub(crate) struct BaggageField { #[derive(Clone, Debug)] pub(crate) struct OverseerInfo { /// Where the support crate `::polkadot_overseer_gen` lives. - pub(crate) support_crate_name: TokenStream, + pub(crate) support_crate: TokenStream, /// Fields annotated with `#[subsystem(..)]`. pub(crate) subsystems: Vec, @@ -297,7 +297,7 @@ pub(crate) struct OverseerInfo { impl OverseerInfo { pub(crate) fn support_crate_name(&self) -> &TokenStream { - &self.support_crate_name + &self.support_crate } pub(crate) fn variant_names(&self) -> Vec { diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs index a4cd96882909..450ac61b4fc0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs @@ -22,12 +22,22 @@ use syn::{ parse::{Parse, ParseBuffer}, punctuated::Punctuated, spanned::Spanned, - Error, Path, Result, Token, + Error, Ident, Path, Result, Token, }; #[derive(Clone, Debug)] enum SubsystemAttrItem { - ExternErrorType { tag: kw::error, eq_token: Token![=], value: Path }, + /// Error type provided by the user. + ExternErrorType { + tag: kw::error, + eq_token: Token![=], + value: Path, + }, + Subsystem { + tag: Option, + eq_token: Option, + value: Ident, + }, } impl ToTokens for SubsystemAttrItem { @@ -36,6 +46,9 @@ impl ToTokens for SubsystemAttrItem { Self::ExternErrorType { tag, eq_token, value } => { quote! { #tag #eq_token, #value } }, + Self::Subsystem { tag, eq_token, value } => { + quote! { #tag #eq_token, #value } + }, }; tokens.extend(ts.into_iter()); } @@ -50,17 +63,24 @@ impl Parse for SubsystemAttrItem { eq_token: input.parse()?, value: input.parse()?, }) + } else if lookahead.peek(kw::subsystem) { + Ok(SubsystemAttrItem::Subsystem { + tag: Some(input.parse::()?), + eq_token: Some(input.parse()?), + value: input.parse()?, + }) } else { - Err(lookahead.error()) + Ok(SubsystemAttrItem::Subsystem { tag: None, eq_token: None, value: input.parse()? }) } } } -/// Attribute arguments +/// Attribute arguments `$args` in `#[subsystem( $args )]`. #[derive(Clone, Debug)] pub(crate) struct SubsystemAttrArgs { span: Span, pub(crate) extern_error_ty: Path, + pub(crate) subsystem_ident: Ident, } impl Spanned for SubsystemAttrArgs { @@ -77,10 +97,9 @@ macro_rules! extract_variant { extract_variant!($unique, $variant).ok_or_else(|| Error::new(Span::call_site(), $err)) }; ($unique:expr, $variant:ident) => { - $unique.values().find_map(|item| { - match item { - SubsystemAttrItem::$variant { value, .. } => Some(value.clone()), - } + $unique.values().find_map(|item| match item { + SubsystemAttrItem::$variant { value, .. } => Some(value.clone()), + _ => None, }) }; } @@ -106,7 +125,8 @@ impl Parse for SubsystemAttrArgs { return Err(e) } } - let error = extract_variant!(unique, ExternErrorType; err = "Must annotate the identical overseer error type via `error=..`.")?; - Ok(SubsystemAttrArgs { span, extern_error_ty: error }) + let extern_error_ty = extract_variant!(unique, ExternErrorType; err = "Must annotate the identical overseer error type via `error=..`.")?; + let subsystem_ident = extract_variant!(unique, Subsystem; err = "Must annotate the identical overseer error type via `subsystem=..` or plainly as `Subsystem` as specified in the overseer declaration.")?; + Ok(SubsystemAttrArgs { span, extern_error_ty, subsystem_ident }) } } diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs new file mode 100644 index 000000000000..0286ed71cec8 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -0,0 +1,72 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use proc_macro2::{Ident, Span, TokenStream}; +use quote::{format_ident, quote, ToTokens}; +use syn::{parse2, parse_quote, punctuated::Punctuated, spanned::Spanned, Result}; + +use super::{parse::*, *}; + +pub(crate) fn impl_subsystem_gen( + attr: TokenStream, + orig: TokenStream, +) -> Result { + let SubsystemAttrArgs { extern_error_ty: error_ty, subsystem_ident, .. } = + parse2::(attr.clone())?; + + let mut struktured_impl: syn::ItemImpl = parse2(orig)?; + + let support_crate = support_crate(); + + let me = match &*struktured_impl.self_ty { + syn::Type::Path(ref path) => dbg!(path) + .path + .segments + .last() + .expect("Must be ident. FIXME bail properly") + .ident + .clone(), + _ => return Err(syn::Error::new(attr.span(), "Doesn't take any arguments at this time")), + }; + let subsystem_ctx_trait = format_ident!("{}ContextTrait", subsystem_ident); + let subsystem_sender_trait = format_ident!("{}SenderTrait", subsystem_ident); + let extra_where_predicates: Punctuated = parse_quote! { + Context: #subsystem_ctx_trait, + Context: #support_crate::SubsystemContext, + ::Sender: #subsystem_sender_trait, + ::Sender: #subsystem_sender_trait, + }; + struktured_impl.trait_.replace(( + None, + parse_quote! { + #support_crate::Subsystem + }, + syn::token::For::default(), + )); + let where_clause = struktured_impl.generics.make_where_clause(); + where_clause.predicates.extend(extra_where_predicates); + + let ts = + expander::Expander::new(format!("subsystem-{}-expansion", me.to_string().to_lowercase())) + .add_comment("Generated overseer code by `#[subsystem(..)]`".to_owned()) + .dry(!cfg!(feature = "expand")) + .verbose(true) + .fmt(expander::Edition::_2021) + .write_to_out_dir(struktured_impl.into_token_stream()) + .expect("Expander does not fail due to IO in OUT_DIR. qed"); + + Ok(ts) +} From d1d84512c4835ab94de32939ce391c75e8972b01 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 2 May 2022 17:28:12 +0200 Subject: [PATCH 085/139] compilation fixes (incomplete) --- node/network/bridge/src/lib.rs | 5 ++++- node/overseer/overseer-gen/proc-macro/src/subsystem.rs | 9 +++++++-- node/overseer/src/lib.rs | 6 +++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 2faf76181b4f..14921059c011 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -38,7 +38,9 @@ use polkadot_node_subsystem::{ errors::{SubsystemError, SubsystemResult}, messages::{ network_bridge_event::{NewGossipTopology, TopologyPeerInfo}, - AllMessages, CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, + AllMessages, ApprovalDistributionMessage, BitfieldDistributionMessage, + CollatorProtocolMessage, GossipSupportMessage, NetworkBridgeEvent, NetworkBridgeMessage, + StatementDistributionMessage, }, overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemSender, @@ -1099,6 +1101,7 @@ async fn dispatch_validation_events_to_all( I: IntoIterator>, I::IntoIter: Send, { + let sender = ctx.sender(); for event in events { sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 0286ed71cec8..2d7a2a98d721 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -32,11 +32,16 @@ pub(crate) fn impl_subsystem_gen( let support_crate = support_crate(); let me = match &*struktured_impl.self_ty { - syn::Type::Path(ref path) => dbg!(path) + syn::Type::Path(ref path) => path .path .segments .last() - .expect("Must be ident. FIXME bail properly") + .ok_or_else(|| { + syn::Error::new( + path.path.span(), + "Missing an identifier at the end of the path provided", + ) + })? .ident .clone(), _ => return Err(syn::Error::new(attr.span(), "Doesn't take any arguments at this time")), diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 7f610ab7c2b4..f4b7d57cf9b7 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -108,9 +108,9 @@ use parity_util_mem::MemoryAllocationTracker; pub use polkadot_overseer_gen as gen; pub use polkadot_overseer_gen::{ - overlord, FromOverseer, MapSubsystem, MessagePacket, SignalsReceived, SpawnNamed, Subsystem, - SubsystemContext, SubsystemIncomingMessages, SubsystemInstance, SubsystemMeterReadouts, - SubsystemMeters, SubsystemSender, TimeoutExt, ToOverseer, + overlord, subsystem, FromOverseer, MapSubsystem, MessagePacket, SignalsReceived, SpawnNamed, + Subsystem, SubsystemContext, SubsystemIncomingMessages, SubsystemInstance, + SubsystemMeterReadouts, SubsystemMeters, SubsystemSender, TimeoutExt, ToOverseer, }; /// Store 2 days worth of blocks, not accounting for forks, From 20c57264a578104f53c400b3e212271d3203584c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 14:41:34 +0200 Subject: [PATCH 086/139] make `#[subsystem]` annotations work across multiple fiels --- node/collation-generation/src/lib.rs | 2 +- node/core/av-store/src/lib.rs | 2 +- node/core/candidate-validation/src/lib.rs | 2 +- node/core/dispute-coordinator/src/lib.rs | 2 +- node/network/approval-distribution/src/lib.rs | 6 +- .../availability-distribution/src/lib.rs | 2 +- node/network/availability-recovery/src/lib.rs | 2 +- node/network/bitfield-distribution/src/lib.rs | 2 +- node/network/collator-protocol/src/lib.rs | 2 +- .../proc-macro/src/impl_channels_out.rs | 2 + .../proc-macro/src/impl_overseer.rs | 1 + .../src/impl_subsystem_ctx_sender.rs | 5 +- .../overseer-gen/proc-macro/src/lib.rs | 18 +++--- .../overseer-gen/proc-macro/src/overseer.rs | 7 +-- .../overseer-gen/proc-macro/src/parse/mod.rs | 2 +- .../src/parse/parse_overseer_struct.rs | 4 +- .../src/parse/parse_subsystem_attr.rs | 36 ++++++++++-- .../overseer-gen/proc-macro/src/subsystem.rs | 55 ++++++++++++++++--- node/overseer/src/lib.rs | 3 +- 19 files changed, 108 insertions(+), 47 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index 74be0548888c..c3b7115fc7cc 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -164,7 +164,7 @@ impl CollationGenerationSubsystem { } } -#[overseer::subsystem(CollationGeneration, error=SubsystemError)] +#[overseer::subsystem(CollationGeneration, error=SubsystemError, prefix=self::overseer)] impl CollationGenerationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 1c4fb859710c..d443cda6265f 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -519,7 +519,7 @@ impl KnownUnfinalizedBlocks { } } -#[overseer::subsystem(AvailabilityStore, error=SubsystemError)] +#[overseer::subsystem(AvailabilityStore, error=SubsystemError, prefix=self::overseer)] impl AvailabilityStoreSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(self, ctx).map(|_| Ok(())).boxed(); diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index c87cc1b9cc9b..9399f61bc0bb 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -93,7 +93,7 @@ impl CandidateValidationSubsystem { } } -#[overseer::subsystem(CandidateValidation, error=SubsystemError)] +#[overseer::subsystem(CandidateValidation, error=SubsystemError, prefix=self::overseer)] impl CandidateValidationSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run( diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index f1a36c3305e0..32c78bfa7447 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -124,7 +124,7 @@ impl Config { } } -#[overseer::subsystem(DisputeCoordinator, error=SubsystemError)] +#[overseer::subsystem(DisputeCoordinator, error=SubsystemError, prefix=self::overseer)] impl DisputeCoordinatorSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async { diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index f1d05d16ec03..692b00056427 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -720,7 +720,7 @@ impl State { Context: overseer::ApprovalDistributionContextTrait, ::Sender: overseer::ApprovalDistributionSenderTrait, - R: CyrptoRng + Rng, + R: CryptoRng + Rng, { let block_hash = assignment.block_hash.clone(); let validator_index = assignment.validator; @@ -1730,8 +1730,8 @@ impl ApprovalDistribution { } } -#[overseer::subsystem(error=SubsystemError)] -impl ApprovalDistribution { +#[overseer::subsystem(ApprovalDistribution, error=SubsystemError, prefix=self::overseer)] +impl ApprovalDistributionSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index bc546b18df04..ae379cbb330a 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -68,7 +68,7 @@ pub struct IncomingRequestReceivers { pub chunk_req_receiver: IncomingRequestReceiver, } -#[overseer::subsystem(AvailabilityDistribution, error=SubsystemError)] +#[overseer::subsystem(AvailabilityDistribution, error=SubsystemError, prefix=self::overseer)] impl AvailabilityDistributionSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index e19557304d7e..67b3dab30fd7 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -806,7 +806,7 @@ impl Default for State { } } -#[overseer::subsystem(AvailabilityRecovery, error=SubsystemError)] +#[overseer::subsystem(AvailabilityRecovery, error=SubsystemError, prefix=self::overseer)] impl AvailabilityRecoverySubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 32469d660211..95447e872415 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -788,7 +788,7 @@ async fn send_tracked_gossip_message( .await; } -#[overseer::subsystem(error=SubsystemError)] +#[overseer::subsystem(BitfieldDistribution, error=SubsystemError, prefix=self::overseer)] impl BitfieldDistribution { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 75e6e903097e..fe57017b6f60 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -113,7 +113,7 @@ impl CollatorProtocolSubsystem { } } -#[overseer::subsystem(CollatorProtocol, error=SubsystemError)] +#[overseer::subsystem(CollatorProtocol, error=SubsystemError, prefix=self::overseer)] impl CollatorProtocolSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 2ef1c20565a6..27f3aef696bf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -80,6 +80,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), + #[allow(unreachable_patterns)] // And everything that's not WIP but no subsystem consumes it unused_msg => { #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); @@ -119,6 +120,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), // And everything that's not WIP but no subsystem consumes it + #[allow(unreachable_patterns)] unused_msg => { #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); Ok(()) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 113bbdcec421..24b3d4be0dd2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -150,6 +150,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> proc_macro2::TokenStr #message_wrapper :: Empty => {} // And everything that's not WIP but no subsystem consumes it + #[allow(unreachable_patterns)] unused_msg => { #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index d7731be8dced..bbcd11d0b67d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -119,7 +119,8 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { #outgoing_wrapper :: #outgoing_variant ( msg ) => #all_messages_wrapper :: #subsystem_generic ( msg ), )* #outgoing_wrapper :: Empty => #all_messages_wrapper :: Empty, - // And everything that's not WIP but no subsystem consumes it + // And everything that's not WIP but no subsystem consumes it + #[allow(unreachable_patterns)] unused_msg => { #support_crate :: gum :: warn!("Nothing consumes {:?}", unused_msg); #all_messages_wrapper :: Empty @@ -241,7 +242,7 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu } pub(crate) fn impl_subsystem_sender( - support_crate: &TokenStream, + support_crate: &Path, outgoing_wrappers: impl IntoIterator, all_messages_wrapper: &Ident, subsystem_sender_name: &Ident, diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 2d7347bce675..3301966a4565 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -17,8 +17,7 @@ #![deny(unused_crate_dependencies)] use proc_macro2::{Ident, Span, TokenStream}; -use quote::{format_ident, quote, ToTokens}; -use syn::{parse2, parse_quote, punctuated::Punctuated, spanned::Spanned, Result}; +use syn::{parse_quote, spanned::Spanned, Path}; mod impl_builder; mod impl_channels_out; @@ -42,18 +41,17 @@ use parse::*; use self::{overseer::*, subsystem::*}; /// Obtain the support crate `Path` as `TokenStream`. -pub(crate) fn support_crate() -> TokenStream { - if cfg!(test) { - quote! {crate} +pub(crate) fn support_crate() -> Result { + Ok(if cfg!(test) { + parse_quote! {crate} } else { use proc_macro_crate::{crate_name, FoundCrate}; - let crate_name = crate_name("polkadot-overseer-gen") - .expect("Support crate `polkadot-overseer-gen` is present in `Cargo.toml`. qed"); + let crate_name = crate_name("polkadot-overseer-gen")?; match crate_name { - FoundCrate::Itself => quote! {crate}, - FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(), + FoundCrate::Itself => parse_quote! {crate}, + FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).into(), } - } + }) } #[proc_macro_attribute] diff --git a/node/overseer/overseer-gen/proc-macro/src/overseer.rs b/node/overseer/overseer-gen/proc-macro/src/overseer.rs index b3aee62428c1..54944053bbb6 100644 --- a/node/overseer/overseer-gen/proc-macro/src/overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/overseer.rs @@ -14,9 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use proc_macro2::{Ident, Span, TokenStream}; -use quote::{format_ident, quote, ToTokens}; -use syn::{parse2, parse_quote, punctuated::Punctuated, spanned::Spanned, Result}; +use proc_macro2::TokenStream; +use syn::{parse2, Result}; use super::{parse::*, *}; @@ -29,7 +28,7 @@ pub(crate) fn impl_overseer_gen( let of: OverseerGuts = parse2(orig)?; - let support_crate = support_crate(); + let support_crate = support_crate().expect("The crate this macro is run for, includes the proc-macro support as dependency, otherwise it could not be run in the first place. qed"); let info = OverseerInfo { support_crate, subsystems: of.subsystems, diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs index e5166464ffdf..f1d82ff93451 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/mod.rs @@ -15,7 +15,6 @@ // along with Polkadot. If not, see . mod kw { - /// An external event. syn::custom_keyword!(event); syn::custom_keyword!(signal); syn::custom_keyword!(error); @@ -24,6 +23,7 @@ mod kw { syn::custom_keyword!(signal_capacity); syn::custom_keyword!(message_capacity); syn::custom_keyword!(subsystem); + syn::custom_keyword!(prefix); } mod parse_overseer_attr; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs index 317b47d129ea..38c26a8cf377 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs @@ -263,7 +263,7 @@ pub(crate) struct BaggageField { #[derive(Clone, Debug)] pub(crate) struct OverseerInfo { /// Where the support crate `::polkadot_overseer_gen` lives. - pub(crate) support_crate: TokenStream, + pub(crate) support_crate: Path, /// Fields annotated with `#[subsystem(..)]`. pub(crate) subsystems: Vec, @@ -296,7 +296,7 @@ pub(crate) struct OverseerInfo { } impl OverseerInfo { - pub(crate) fn support_crate_name(&self) -> &TokenStream { + pub(crate) fn support_crate_name(&self) -> &Path { &self.support_crate } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs index 450ac61b4fc0..b4bf7c55722b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs @@ -28,22 +28,38 @@ use syn::{ #[derive(Clone, Debug)] enum SubsystemAttrItem { /// Error type provided by the user. - ExternErrorType { + ErrorType { tag: kw::error, eq_token: Token![=], value: Path, }, + /// For which slot in the overseer this should be plugged. + /// + /// The subsystem implementation can and should have a different name + /// from the declared parameter type in the overseer. Subsystem { tag: Option, eq_token: Option, value: Ident, }, + /// The prefix to apply when a subsystem is implemented in a different file/crate + /// than the overseer itself. + /// + /// Important for `#[subsystem(..)]` to reference the traits correctly. + TraitPrefix { + tag: kw::prefix, + eq_token: Token![=], + value: Path, + }, } impl ToTokens for SubsystemAttrItem { fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { let ts = match self { - Self::ExternErrorType { tag, eq_token, value } => { + Self::TraitPrefix { tag, eq_token, value } => { + quote! { #tag #eq_token, #value } + }, + Self::ErrorType { tag, eq_token, value } => { quote! { #tag #eq_token, #value } }, Self::Subsystem { tag, eq_token, value } => { @@ -58,11 +74,17 @@ impl Parse for SubsystemAttrItem { fn parse(input: &ParseBuffer) -> Result { let lookahead = input.lookahead1(); if lookahead.peek(kw::error) { - Ok(SubsystemAttrItem::ExternErrorType { + Ok(SubsystemAttrItem::ErrorType { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, }) + } else if lookahead.peek(kw::prefix) { + Ok(SubsystemAttrItem::TraitPrefix { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) } else if lookahead.peek(kw::subsystem) { Ok(SubsystemAttrItem::Subsystem { tag: Some(input.parse::()?), @@ -79,8 +101,9 @@ impl Parse for SubsystemAttrItem { #[derive(Clone, Debug)] pub(crate) struct SubsystemAttrArgs { span: Span, - pub(crate) extern_error_ty: Path, + pub(crate) error_ty: Path, pub(crate) subsystem_ident: Ident, + pub(crate) trait_prefix_path: Option, } impl Spanned for SubsystemAttrArgs { @@ -125,8 +148,9 @@ impl Parse for SubsystemAttrArgs { return Err(e) } } - let extern_error_ty = extract_variant!(unique, ExternErrorType; err = "Must annotate the identical overseer error type via `error=..`.")?; + let error_ty = extract_variant!(unique, ErrorType; err = "Must annotate the identical overseer error type via `error=..`.")?; let subsystem_ident = extract_variant!(unique, Subsystem; err = "Must annotate the identical overseer error type via `subsystem=..` or plainly as `Subsystem` as specified in the overseer declaration.")?; - Ok(SubsystemAttrArgs { span, extern_error_ty, subsystem_ident }) + let trait_prefix_path = extract_variant!(unique, TraitPrefix); + Ok(SubsystemAttrArgs { span, error_ty, subsystem_ident, trait_prefix_path }) } } diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 2d7a2a98d721..002f9d0b6793 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -14,9 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use proc_macro2::{Ident, Span, TokenStream}; -use quote::{format_ident, quote, ToTokens}; -use syn::{parse2, parse_quote, punctuated::Punctuated, spanned::Spanned, Result}; +use proc_macro2::TokenStream; +use quote::{format_ident, ToTokens}; +use syn::{parse2, parse_quote, punctuated::Punctuated, Result}; use super::{parse::*, *}; @@ -24,13 +24,36 @@ pub(crate) fn impl_subsystem_gen( attr: TokenStream, orig: TokenStream, ) -> Result { - let SubsystemAttrArgs { extern_error_ty: error_ty, subsystem_ident, .. } = - parse2::(attr.clone())?; + let args = parse2::(attr.clone())?; + let _span = args.span(); + let SubsystemAttrArgs { + error_ty, + subsystem_ident, + trait_prefix_path + , .. } = + args; let mut struktured_impl: syn::ItemImpl = parse2(orig)?; - let support_crate = support_crate(); + // always prefer the direct usage, if it's not there, let's see if there is + // a `prefix=*` provided. Either is ok. + // TODO: technically this is two different things: + // The place where the `#[overlord]` is annotated is where all `trait *SenderTrait` and + // `trait *ContextTrait` types exist. + // The other usage is the true support crate `polkadot-overseer-gen`, where the static ones + // are declared. + // Right now, if the `support_crate` is not included, it falls back silently to the `trait_prefix_path`. + let support_crate = support_crate() + .or_else(|_e| { + trait_prefix_path.clone().ok_or_else(|| { + syn::Error::new(attr.span(), "Couldn't find `polkadot-overseer-gen` in manifest, but also missing a `prefix=` to help trait bound resolution") + }) + })?; + let trait_prefix_path = trait_prefix_path.unwrap_or_else(|| parse_quote!{ self }); + if trait_prefix_path.segments.trailing_punct() { + return Err(syn::Error::new(trait_prefix_path.span(), "Must not end with `::`")) + } let me = match &*struktured_impl.self_ty { syn::Type::Path(ref path) => path .path @@ -49,10 +72,10 @@ pub(crate) fn impl_subsystem_gen( let subsystem_ctx_trait = format_ident!("{}ContextTrait", subsystem_ident); let subsystem_sender_trait = format_ident!("{}SenderTrait", subsystem_ident); let extra_where_predicates: Punctuated = parse_quote! { - Context: #subsystem_ctx_trait, + Context: #trait_prefix_path::#subsystem_ctx_trait, Context: #support_crate::SubsystemContext, - ::Sender: #subsystem_sender_trait, - ::Sender: #subsystem_sender_trait, + ::Sender: #trait_prefix_path::#subsystem_sender_trait, + ::Sender: #trait_prefix_path::#subsystem_sender_trait, }; struktured_impl.trait_.replace(( None, @@ -75,3 +98,17 @@ pub(crate) fn impl_subsystem_gen( Ok(ts) } + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_path() { + let _p: Path = parse_quote!{ self }; + let _p: Path = parse_quote!{ crate }; + let _p: Path = parse_quote!{ ::foo }; + let _p: Path = parse_quote!{ bar }; + } +} diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index f4b7d57cf9b7..0bcf779e39c9 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -77,14 +77,13 @@ use polkadot_primitives::{ }; use sp_api::{ApiExt, ProvideRuntimeApi}; -use polkadot_node_network_protocol::VersionedValidationProtocol; use polkadot_node_subsystem_types::messages::{ ApprovalDistributionMessage, ApprovalVotingMessage, AvailabilityDistributionMessage, AvailabilityRecoveryMessage, AvailabilityStoreMessage, BitfieldDistributionMessage, BitfieldSigningMessage, CandidateBackingMessage, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, CollationGenerationMessage, CollatorProtocolMessage, DisputeCoordinatorMessage, DisputeDistributionMessage, GossipSupportMessage, - NetworkBridgeEvent, NetworkBridgeMessage, ProvisionerMessage, PvfCheckerMessage, + NetworkBridgeMessage, ProvisionerMessage, PvfCheckerMessage, RuntimeApiMessage, StatementDistributionMessage, }; pub use polkadot_node_subsystem_types::{ From b9bbe5939b35e11db4c7df374605d45f0e2e3c65 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 14:42:27 +0200 Subject: [PATCH 087/139] fix imports --- node/core/backing/src/lib.rs | 7 +++---- node/network/approval-distribution/src/lib.rs | 2 +- .../dispute-distribution/src/receiver/mod.rs | 2 +- .../dispute-distribution/src/sender/mod.rs | 3 +-- .../src/parse/parse_subsystem_attr.rs | 18 +++--------------- .../overseer-gen/proc-macro/src/subsystem.rs | 18 ++++++------------ node/overseer/src/lib.rs | 4 ++-- 7 files changed, 17 insertions(+), 37 deletions(-) diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index d0e15e9e412c..04808a18ba13 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -37,10 +37,9 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::{ jaeger, messages::{ - AvailabilityDistributionMessage, AvailabilityStoreMessage, - CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, - DisputeCoordinatorMessage, ProvisionableData, ProvisionerMessage, RuntimeApiRequest, - StatementDistributionMessage, + AvailabilityDistributionMessage, AvailabilityStoreMessage, CandidateBackingMessage, + CandidateValidationMessage, CollatorProtocolMessage, DisputeCoordinatorMessage, + ProvisionableData, ProvisionerMessage, RuntimeApiRequest, StatementDistributionMessage, }, overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, PerLeafSpan, SpawnedSubsystem, Stage, SubsystemContext, SubsystemError, SubsystemSender, diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 692b00056427..338f28ecde56 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -511,7 +511,7 @@ impl State { ) where Context: overseer::ApprovalDistributionContextTrait, ::Sender: - overseer::ApprovalDistributionSenderTrait, + overseer::ApprovalDistributionSenderTrait, { self.topologies.insert_topology(session, topology); let topology = self.topologies.get_topology(session).expect("just inserted above; qed"); diff --git a/node/network/dispute-distribution/src/receiver/mod.rs b/node/network/dispute-distribution/src/receiver/mod.rs index f0d9cc8262eb..82554569aba6 100644 --- a/node/network/dispute-distribution/src/receiver/mod.rs +++ b/node/network/dispute-distribution/src/receiver/mod.rs @@ -40,7 +40,7 @@ use polkadot_node_network_protocol::{ }; use polkadot_node_primitives::DISPUTE_WINDOW; use polkadot_node_subsystem::{ - messages::{AllMessages, DisputeCoordinatorMessage, ImportStatementsResult}, + messages::{DisputeCoordinatorMessage, ImportStatementsResult}, overseer, SubsystemSender, }; use polkadot_node_subsystem_util::{runtime, runtime::RuntimeInfo}; diff --git a/node/network/dispute-distribution/src/sender/mod.rs b/node/network/dispute-distribution/src/sender/mod.rs index 80684c9bd0b7..80dbb0414a5b 100644 --- a/node/network/dispute-distribution/src/sender/mod.rs +++ b/node/network/dispute-distribution/src/sender/mod.rs @@ -21,8 +21,7 @@ use futures::channel::{mpsc, oneshot}; use polkadot_node_network_protocol::request_response::v1::DisputeRequest; use polkadot_node_primitives::{CandidateVotes, DisputeMessage, SignedDisputeStatement}; use polkadot_node_subsystem::{ - messages::{AllMessages, DisputeCoordinatorMessage}, - overseer, ActiveLeavesUpdate, SubsystemContext, + messages::DisputeCoordinatorMessage, overseer, ActiveLeavesUpdate, SubsystemContext, }; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::v2::{CandidateHash, DisputeStatement, Hash, SessionIndex}; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs index b4bf7c55722b..28286d0e645c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs @@ -28,29 +28,17 @@ use syn::{ #[derive(Clone, Debug)] enum SubsystemAttrItem { /// Error type provided by the user. - ErrorType { - tag: kw::error, - eq_token: Token![=], - value: Path, - }, + ErrorType { tag: kw::error, eq_token: Token![=], value: Path }, /// For which slot in the overseer this should be plugged. /// /// The subsystem implementation can and should have a different name /// from the declared parameter type in the overseer. - Subsystem { - tag: Option, - eq_token: Option, - value: Ident, - }, + Subsystem { tag: Option, eq_token: Option, value: Ident }, /// The prefix to apply when a subsystem is implemented in a different file/crate /// than the overseer itself. /// /// Important for `#[subsystem(..)]` to reference the traits correctly. - TraitPrefix { - tag: kw::prefix, - eq_token: Token![=], - value: Path, - }, + TraitPrefix { tag: kw::prefix, eq_token: Token![=], value: Path }, } impl ToTokens for SubsystemAttrItem { diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 002f9d0b6793..7458d6dcb922 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -26,12 +26,7 @@ pub(crate) fn impl_subsystem_gen( ) -> Result { let args = parse2::(attr.clone())?; let _span = args.span(); - let SubsystemAttrArgs { - error_ty, - subsystem_ident, - trait_prefix_path - , .. } = - args; + let SubsystemAttrArgs { error_ty, subsystem_ident, trait_prefix_path, .. } = args; let mut struktured_impl: syn::ItemImpl = parse2(orig)?; @@ -50,7 +45,7 @@ pub(crate) fn impl_subsystem_gen( }) })?; - let trait_prefix_path = trait_prefix_path.unwrap_or_else(|| parse_quote!{ self }); + let trait_prefix_path = trait_prefix_path.unwrap_or_else(|| parse_quote! { self }); if trait_prefix_path.segments.trailing_punct() { return Err(syn::Error::new(trait_prefix_path.span(), "Must not end with `::`")) } @@ -99,16 +94,15 @@ pub(crate) fn impl_subsystem_gen( Ok(ts) } - #[cfg(test)] mod tests { use super::*; #[test] fn is_path() { - let _p: Path = parse_quote!{ self }; - let _p: Path = parse_quote!{ crate }; - let _p: Path = parse_quote!{ ::foo }; - let _p: Path = parse_quote!{ bar }; + let _p: Path = parse_quote! { self }; + let _p: Path = parse_quote! { crate }; + let _p: Path = parse_quote! { ::foo }; + let _p: Path = parse_quote! { bar }; } } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 0bcf779e39c9..a65d0e8e282c 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -83,8 +83,8 @@ use polkadot_node_subsystem_types::messages::{ BitfieldSigningMessage, CandidateBackingMessage, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, CollationGenerationMessage, CollatorProtocolMessage, DisputeCoordinatorMessage, DisputeDistributionMessage, GossipSupportMessage, - NetworkBridgeMessage, ProvisionerMessage, PvfCheckerMessage, - RuntimeApiMessage, StatementDistributionMessage, + NetworkBridgeMessage, ProvisionerMessage, PvfCheckerMessage, RuntimeApiMessage, + StatementDistributionMessage, }; pub use polkadot_node_subsystem_types::{ errors::{SubsystemError, SubsystemResult}, From ce49525b4d7711c321bc8e0fc6906579aa42131e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 14:42:40 +0200 Subject: [PATCH 088/139] account --- runtime/parachains/src/paras_inherent/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/parachains/src/paras_inherent/mod.rs b/runtime/parachains/src/paras_inherent/mod.rs index e6a4f6e7a6aa..1832e05d9527 100644 --- a/runtime/parachains/src/paras_inherent/mod.rs +++ b/runtime/parachains/src/paras_inherent/mod.rs @@ -535,7 +535,7 @@ impl Pallet { // Give some time slice to dispatch pending upward messages. // this is max config.ump_service_total_weight - let _ump_weight = >::process_pending_upward_messages(); + let total_consumed_weight = total_consumed_weight + >::process_pending_upward_messages(); METRICS.on_after_filter(total_consumed_weight); From 3ce7b200bb58b3b60a8638828c0bcf26c642285a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 15:41:28 +0200 Subject: [PATCH 089/139] start killing all the stupid trait bounds that are generated --- node/network/dispute-distribution/src/lib.rs | 34 ++--- .../dispute-distribution/src/sender/mod.rs | 53 ++------ .../src/sender/send_task.rs | 30 +---- .../proc-macro/src/impl_channels_out.rs | 1 + .../overseer-gen/proc-macro/src/lib.rs | 14 ++- .../src/parse/parse_subsystem_attr.rs | 12 +- .../overseer-gen/proc-macro/src/subsystem.rs | 118 ++++++++++++------ node/overseer/overseer-gen/src/lib.rs | 4 +- node/overseer/src/lib.rs | 4 +- runtime/parachains/src/paras_inherent/mod.rs | 2 +- 10 files changed, 129 insertions(+), 143 deletions(-) diff --git a/node/network/dispute-distribution/src/lib.rs b/node/network/dispute-distribution/src/lib.rs index 65ae2984cb31..93a4536f4d28 100644 --- a/node/network/dispute-distribution/src/lib.rs +++ b/node/network/dispute-distribution/src/lib.rs @@ -114,9 +114,9 @@ pub struct DisputeDistributionSubsystem { metrics: Metrics, } -impl overseer::Subsystem for DisputeDistributionSubsystem +#[overseer::subsystem(DisputeDistribution, error = SubsystemError, prefix = self::overseer)] +impl DisputeDistributionSubsystem where - Context: overseer::DisputeDistributionContextTrait, ::Sender: overseer::DisputeDistributionSenderTrait + Sync + Send, AD: AuthorityDiscovery + Clone, @@ -131,6 +131,7 @@ where } } +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] impl DisputeDistributionSubsystem where AD: AuthorityDiscovery + Clone, @@ -159,12 +160,7 @@ where } /// Start processing work as passed on from the Overseer. - async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait + Sync + Send, - { + async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> { let receiver = DisputesReceiver::new( ctx.sender().clone(), self.req_receiver @@ -207,12 +203,7 @@ where &mut self, ctx: &mut Context, signal: OverseerSignal, - ) -> Result - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result { match signal { OverseerSignal::Conclude => return Ok(SignalResult::Conclude), OverseerSignal::ActiveLeaves(update) => { @@ -228,12 +219,7 @@ where &mut self, ctx: &mut Context, msg: DisputeDistributionMessage, - ) -> Result<()> - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result<()> { match msg { DisputeDistributionMessage::SendDispute(dispute_msg) => self.disputes_sender.start_sender(ctx, &mut self.runtime, dispute_msg).await?, @@ -251,16 +237,12 @@ enum MuxedMessage { Sender(Option), } +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] impl MuxedMessage { async fn receive( ctx: &mut Context, from_sender: &mut mpsc::Receiver, - ) -> Self - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Self { // We are only fusing here to make `select` happy, in reality we will quit if the stream // ends. let from_overseer = ctx.recv().fuse(); diff --git a/node/network/dispute-distribution/src/sender/mod.rs b/node/network/dispute-distribution/src/sender/mod.rs index 80dbb0414a5b..975c69b0fe30 100644 --- a/node/network/dispute-distribution/src/sender/mod.rs +++ b/node/network/dispute-distribution/src/sender/mod.rs @@ -65,6 +65,7 @@ pub struct DisputeSender { metrics: Metrics, } +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] impl DisputeSender { /// Create a new `DisputeSender` which can be used to start dispute sendings. pub fn new(tx: mpsc::Sender, metrics: Metrics) -> Self { @@ -83,12 +84,7 @@ impl DisputeSender { ctx: &mut Context, runtime: &mut RuntimeInfo, msg: DisputeMessage, - ) -> Result<()> - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result<()> { let req: DisputeRequest = msg.into(); let candidate_hash = req.0.candidate_receipt.hash(); match self.disputes.entry(candidate_hash) { @@ -123,12 +119,7 @@ impl DisputeSender { ctx: &mut Context, runtime: &mut RuntimeInfo, update: ActiveLeavesUpdate, - ) -> Result<()> - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result<()> { let ActiveLeavesUpdate { activated, deactivated } = update; let deactivated: HashSet<_> = deactivated.into_iter().collect(); self.active_heads.retain(|h| !deactivated.contains(h)); @@ -193,12 +184,7 @@ impl DisputeSender { ctx: &mut Context, runtime: &mut RuntimeInfo, dispute: (SessionIndex, CandidateHash), - ) -> Result<()> - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result<()> { let (session_index, candidate_hash) = dispute; // A relay chain head is required as context for receiving session info information from runtime and // storage. We will iterate `active_sessions` to find a suitable head. We assume that there is at @@ -321,12 +307,7 @@ impl DisputeSender { &mut self, ctx: &mut Context, runtime: &mut RuntimeInfo, - ) -> Result - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result { let new_sessions = get_active_session_indices(ctx, runtime, &self.active_heads).await?; let new_sessions_raw: HashSet<_> = new_sessions.keys().collect(); let old_sessions_raw: HashSet<_> = self.active_sessions.keys().collect(); @@ -340,16 +321,12 @@ impl DisputeSender { /// Retrieve the currently active sessions. /// /// List is all indices of all active sessions together with the head that was used for the query. +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] async fn get_active_session_indices( ctx: &mut Context, runtime: &mut RuntimeInfo, active_heads: &Vec, -) -> Result> -where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, -{ +) -> Result> { let mut indeces = HashMap::new(); // Iterate all heads we track as active and fetch the child' session indices. for head in active_heads { @@ -360,14 +337,10 @@ where } /// Retrieve Set of active disputes from the dispute coordinator. +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] async fn get_active_disputes( ctx: &mut Context, -) -> JfyiErrorResult> -where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, -{ +) -> JfyiErrorResult> { let (tx, rx) = oneshot::channel(); // Caller scope is in `update_leaves` and this is bounded by fork count. @@ -376,16 +349,12 @@ where } /// Get all locally available dispute votes for a given dispute. +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] async fn get_candidate_votes( ctx: &mut Context, session_index: SessionIndex, candidate_hash: CandidateHash, -) -> JfyiErrorResult> -where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, -{ +) -> JfyiErrorResult> { let (tx, rx) = oneshot::channel(); // Caller scope is in `update_leaves` and this is bounded by fork count. ctx.send_unbounded_message(DisputeCoordinatorMessage::QueryCandidateVotes( diff --git a/node/network/dispute-distribution/src/sender/send_task.rs b/node/network/dispute-distribution/src/sender/send_task.rs index 679765331da1..d5f3a9feb2ec 100644 --- a/node/network/dispute-distribution/src/sender/send_task.rs +++ b/node/network/dispute-distribution/src/sender/send_task.rs @@ -97,6 +97,7 @@ impl TaskResult { } } +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] impl SendTask { /// Initiates sending a dispute message to peers. pub async fn new( @@ -106,12 +107,7 @@ impl SendTask { tx: mpsc::Sender, request: DisputeRequest, metrics: &Metrics, - ) -> Result - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result { let mut send_task = Self { request, deliveries: HashMap::new(), has_failed_sends: false, tx }; send_task.refresh_sends(ctx, runtime, active_sessions, metrics).await?; @@ -128,12 +124,7 @@ impl SendTask { runtime: &mut RuntimeInfo, active_sessions: &HashMap, metrics: &Metrics, - ) -> Result<()> - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result<()> { let new_authorities = self.get_relevant_validators(ctx, runtime, active_sessions).await?; let add_authorities = new_authorities @@ -210,12 +201,7 @@ impl SendTask { ctx: &mut Context, runtime: &mut RuntimeInfo, active_sessions: &HashMap, - ) -> Result> - where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, - { + ) -> Result> { let ref_head = self.request.0.candidate_receipt.descriptor.relay_parent; // Retrieve all authorities which participated in the parachain consensus of the session // in which the candidate was backed. @@ -254,18 +240,14 @@ impl SendTask { /// Start sending of the given message to all given authorities. /// /// And spawn tasks for handling the response. +#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)] async fn send_requests( ctx: &mut Context, tx: mpsc::Sender, receivers: Vec, req: DisputeRequest, metrics: &Metrics, -) -> Result> -where - Context: overseer::DisputeDistributionContextTrait, - ::Sender: - overseer::DisputeDistributionSenderTrait, -{ +) -> Result> { let mut statuses = HashMap::with_capacity(receivers.len()); let mut reqs = Vec::with_capacity(receivers.len()); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 27f3aef696bf..a337a69d46a9 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -65,6 +65,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result = match message { #( #message_wrapper :: #consumes_variant ( inner ) => { diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 3301966a4565..1da58ff5ee2d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -73,7 +73,19 @@ pub fn subsystem( ) -> proc_macro::TokenStream { let attr: TokenStream = attr.into(); let item: TokenStream = item.into(); - impl_subsystem_gen(attr, item) + impl_subsystem_context_trait_bounds(attr, item, MakeSubsystem::ImplSubsystemTrait) + .unwrap_or_else(|err| err.to_compile_error()) + .into() +} + +#[proc_macro_attribute] +pub fn contextbounds( + attr: proc_macro::TokenStream, + item: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + let attr: TokenStream = attr.into(); + let item: TokenStream = item.into(); + impl_subsystem_context_trait_bounds(attr, item, MakeSubsystem::AddContextTraitBounds) .unwrap_or_else(|err| err.to_compile_error()) .into() } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs index 28286d0e645c..6423f0f098fc 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs @@ -28,7 +28,7 @@ use syn::{ #[derive(Clone, Debug)] enum SubsystemAttrItem { /// Error type provided by the user. - ErrorType { tag: kw::error, eq_token: Token![=], value: Path }, + Error { tag: kw::error, eq_token: Token![=], value: Path }, /// For which slot in the overseer this should be plugged. /// /// The subsystem implementation can and should have a different name @@ -47,7 +47,7 @@ impl ToTokens for SubsystemAttrItem { Self::TraitPrefix { tag, eq_token, value } => { quote! { #tag #eq_token, #value } }, - Self::ErrorType { tag, eq_token, value } => { + Self::Error { tag, eq_token, value } => { quote! { #tag #eq_token, #value } }, Self::Subsystem { tag, eq_token, value } => { @@ -62,7 +62,7 @@ impl Parse for SubsystemAttrItem { fn parse(input: &ParseBuffer) -> Result { let lookahead = input.lookahead1(); if lookahead.peek(kw::error) { - Ok(SubsystemAttrItem::ErrorType { + Ok(SubsystemAttrItem::Error { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, @@ -89,7 +89,7 @@ impl Parse for SubsystemAttrItem { #[derive(Clone, Debug)] pub(crate) struct SubsystemAttrArgs { span: Span, - pub(crate) error_ty: Path, + pub(crate) error_path: Option, pub(crate) subsystem_ident: Ident, pub(crate) trait_prefix_path: Option, } @@ -136,9 +136,9 @@ impl Parse for SubsystemAttrArgs { return Err(e) } } - let error_ty = extract_variant!(unique, ErrorType; err = "Must annotate the identical overseer error type via `error=..`.")?; + let error_path = extract_variant!(unique, Error); let subsystem_ident = extract_variant!(unique, Subsystem; err = "Must annotate the identical overseer error type via `subsystem=..` or plainly as `Subsystem` as specified in the overseer declaration.")?; let trait_prefix_path = extract_variant!(unique, TraitPrefix); - Ok(SubsystemAttrArgs { span, error_ty, subsystem_ident, trait_prefix_path }) + Ok(SubsystemAttrArgs { span, error_path, subsystem_ident, trait_prefix_path }) } } diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 7458d6dcb922..3ef1a931547a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -20,15 +20,26 @@ use syn::{parse2, parse_quote, punctuated::Punctuated, Result}; use super::{parse::*, *}; -pub(crate) fn impl_subsystem_gen( +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MakeSubsystem { + /// Impl `trait Subsystem` and apply the trait bounds to the `Context` generic. + /// + /// Relevant to `impl Item` only. + ImplSubsystemTrait, + /// Only apply the trait bounds to the context. + AddContextTraitBounds, +} + +pub(crate) fn impl_subsystem_context_trait_bounds( attr: TokenStream, orig: TokenStream, + make_subsystem: MakeSubsystem, ) -> Result { let args = parse2::(attr.clone())?; - let _span = args.span(); - let SubsystemAttrArgs { error_ty, subsystem_ident, trait_prefix_path, .. } = args; + let span = args.span(); + let SubsystemAttrArgs { error_path, subsystem_ident, trait_prefix_path, .. } = args; - let mut struktured_impl: syn::ItemImpl = parse2(orig)?; + let mut item = parse2::(orig)?; // always prefer the direct usage, if it's not there, let's see if there is // a `prefix=*` provided. Either is ok. @@ -49,49 +60,80 @@ pub(crate) fn impl_subsystem_gen( if trait_prefix_path.segments.trailing_punct() { return Err(syn::Error::new(trait_prefix_path.span(), "Must not end with `::`")) } - let me = match &*struktured_impl.self_ty { - syn::Type::Path(ref path) => path - .path - .segments - .last() - .ok_or_else(|| { - syn::Error::new( - path.path.span(), - "Missing an identifier at the end of the path provided", - ) - })? - .ident - .clone(), - _ => return Err(syn::Error::new(attr.span(), "Doesn't take any arguments at this time")), - }; + let subsystem_ctx_trait = format_ident!("{}ContextTrait", subsystem_ident); let subsystem_sender_trait = format_ident!("{}SenderTrait", subsystem_ident); + let extra_where_predicates: Punctuated = parse_quote! { Context: #trait_prefix_path::#subsystem_ctx_trait, Context: #support_crate::SubsystemContext, ::Sender: #trait_prefix_path::#subsystem_sender_trait, ::Sender: #trait_prefix_path::#subsystem_sender_trait, }; - struktured_impl.trait_.replace(( - None, - parse_quote! { - #support_crate::Subsystem + + let apply_ctx_bound_if_present = move |generics: &mut syn::Generics| -> bool { + if generics + .params + .iter() + .find(|generic| match generic { + syn::GenericParam::Type(ty) if ty.ident == "Context" => true, + _ => false, + }) + .is_some() + { + let where_clause = generics.make_where_clause(); + where_clause.predicates.extend(extra_where_predicates.clone()); + true + } else { + false + } + }; + + match item { + syn::Item::Impl(ref mut struktured_impl) => { + if make_subsystem == MakeSubsystem::ImplSubsystemTrait { + let error_path = error_path.ok_or_else(|| { + syn::Error::new( + span, + "Must annotate the identical overseer error type via `error=..`.", + ) + })?; + // Only replace the subsystem trait if it's desired. + struktured_impl.trait_.replace(( + None, + parse_quote! { + #support_crate::Subsystem + }, + syn::token::For::default(), + )); + } + + apply_ctx_bound_if_present(&mut struktured_impl.generics); + for item in struktured_impl.items.iter_mut() { + match item { + syn::ImplItem::Method(method) => { + apply_ctx_bound_if_present(&mut method.sig.generics); + }, + _others => { + // don't error, just nop + }, + } + } }, - syn::token::For::default(), - )); - let where_clause = struktured_impl.generics.make_where_clause(); - where_clause.predicates.extend(extra_where_predicates); - - let ts = - expander::Expander::new(format!("subsystem-{}-expansion", me.to_string().to_lowercase())) - .add_comment("Generated overseer code by `#[subsystem(..)]`".to_owned()) - .dry(!cfg!(feature = "expand")) - .verbose(true) - .fmt(expander::Edition::_2021) - .write_to_out_dir(struktured_impl.into_token_stream()) - .expect("Expander does not fail due to IO in OUT_DIR. qed"); - - Ok(ts) + syn::Item::Fn(ref mut struktured_fn) => { + if make_subsystem == MakeSubsystem::ImplSubsystemTrait { + return Err(syn::Error::new(struktured_fn.span(), "Cannot make a free function a subsystem, did you mean to apply `contextbound` instead?")) + } + apply_ctx_bound_if_present(&mut struktured_fn.sig.generics); + }, + other => + return Err(syn::Error::new( + other.span(), + "Macro can only be annotated on functions or struct implementations", + )), + }; + + Ok(item.to_token_stream()) } #[cfg(test)] diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 815c8c2122c8..4c6409651936 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -60,9 +60,7 @@ #![deny(missing_docs)] #![deny(unused_crate_dependencies)] -pub use polkadot_overseer_gen_proc_macro::overlord; - -pub use polkadot_overseer_gen_proc_macro::subsystem; +pub use polkadot_overseer_gen_proc_macro::{contextbounds, overlord, subsystem}; #[doc(hidden)] pub use gum; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index a65d0e8e282c..3b7f90e0f383 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -107,8 +107,8 @@ use parity_util_mem::MemoryAllocationTracker; pub use polkadot_overseer_gen as gen; pub use polkadot_overseer_gen::{ - overlord, subsystem, FromOverseer, MapSubsystem, MessagePacket, SignalsReceived, SpawnNamed, - Subsystem, SubsystemContext, SubsystemIncomingMessages, SubsystemInstance, + contextbounds, overlord, subsystem, FromOverseer, MapSubsystem, MessagePacket, SignalsReceived, + SpawnNamed, Subsystem, SubsystemContext, SubsystemIncomingMessages, SubsystemInstance, SubsystemMeterReadouts, SubsystemMeters, SubsystemSender, TimeoutExt, ToOverseer, }; diff --git a/runtime/parachains/src/paras_inherent/mod.rs b/runtime/parachains/src/paras_inherent/mod.rs index 1832e05d9527..e6a4f6e7a6aa 100644 --- a/runtime/parachains/src/paras_inherent/mod.rs +++ b/runtime/parachains/src/paras_inherent/mod.rs @@ -535,7 +535,7 @@ impl Pallet { // Give some time slice to dispatch pending upward messages. // this is max config.ump_service_total_weight - let total_consumed_weight = total_consumed_weight + >::process_pending_upward_messages(); + let _ump_weight = >::process_pending_upward_messages(); METRICS.on_after_filter(total_consumed_weight); From 56e003b838200b58ab63490191df9dd79a09c023 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 16:10:20 +0200 Subject: [PATCH 090/139] add docs to subsystem generation --- .../overseer-gen/proc-macro/src/subsystem.rs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 3ef1a931547a..4e35bca25e77 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -14,6 +14,76 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . + +//! Generates the bounds for a particular subsystem. +//! +//! +//! ## `subsystem` +//! +//! ```rust +//! #[subsystem(Foo, error = Yikes, prefix = somewhere)] +//! impl BarSubsystem { +//! .. +//! } +//! ``` expands to +//! +//! ```rust +//! impl support_crate::Subsystem for BarSubsystem +//! where +//! Context: somewhere::FooSubsystemTrait, +//! Context: support_crate::SubsystemContext, +//! ::Sender: somewhere::FooSenderTrait, +//! ::Sender: somewhere::FooSenderTrait, +//! { +//! } +//! ``` +//! +//! where `support_crate` is either equivalent to `somewhere` or derived from the cargo manifest. +//! +//! +//! ### `contextbounds` +//! +//! ```rust +//! #[contextbounds(Foo, prefix = somewhere)] +//! impl X { +//! .. +//! } +//! ``` +//! +//! expands to +//! +//! ```rust +//! impl X +//! where +//! Context: somewhere::FooSubsystemTrait, +//! Context: support_crate::SubsystemContext, +//! ::Sender: somewhere::FooSenderTrait, +//! ::Sender: somewhere::FooSenderTrait, +//! { +//! } +//! ``` +//! +//! or +//! +//! ```rust +//! #[contextbounds(Foo, prefix = somewhere)] +//! fn do_smth(context: &mut Context) { +//! .. +//! } +//! ``` +//! +//! expands to +//! +//! ```rust +//! fn do_smth(context: &mut Context) +//! where +//! Context: somewhere::FooSubsystemTrait, +//! Context: support_crate::SubsystemContext, +//! ::Sender: somewhere::FooSenderTrait, +//! ::Sender: somewhere::FooSenderTrait, +//! { +//! } +//! ``` use proc_macro2::TokenStream; use quote::{format_ident, ToTokens}; use syn::{parse2, parse_quote, punctuated::Punctuated, Result}; From 225cd7ae73ad31a0e6e6ebf37ee638e9bfe98646 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 16:12:23 +0200 Subject: [PATCH 091/139] headlines --- .../overseer/overseer-gen/proc-macro/src/subsystem.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 4e35bca25e77..0a844dc2eb99 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -14,11 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . - -//! Generates the bounds for a particular subsystem. +//! Generates the bounds for a particular subsystem `Context` and associate `type Sender`. //! //! -//! ## `subsystem` +//! ## Implement `trait Subsystem` via `subsystem` //! //! ```rust //! #[subsystem(Foo, error = Yikes, prefix = somewhere)] @@ -41,7 +40,9 @@ //! where `support_crate` is either equivalent to `somewhere` or derived from the cargo manifest. //! //! -//! ### `contextbounds` +//! ## Add aditional trait bounds for a generic `Context` via `contextbounds` +//! +//! ### To an `ImplItem` //! //! ```rust //! #[contextbounds(Foo, prefix = somewhere)] @@ -63,7 +64,7 @@ //! } //! ``` //! -//! or +//! ### To a free standing `Fn` (not a method, that's covered by the above) //! //! ```rust //! #[contextbounds(Foo, prefix = somewhere)] From 5519fd72d20f2ffe2d225f7fe069941ea8735c8e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 16:44:25 +0200 Subject: [PATCH 092/139] fixup backing after rebase --- node/core/backing/src/lib.rs | 104 +++++++++++++++-------------------- 1 file changed, 43 insertions(+), 61 deletions(-) diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 04808a18ba13..a7e3534071a0 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -130,10 +130,10 @@ impl CandidateBackingSubsystem { } } -impl overseer::Subsystem for CandidateBackingSubsystem +#[overseer::subsystem(CandidateBacking, error = SubsystemError, prefix = self::overseer)] +impl CandidateBackingSubsystem where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: Send + Sync, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { @@ -147,15 +147,12 @@ where } } +#[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] async fn run( mut ctx: Context, keystore: SyncCryptoStorePtr, metrics: Metrics, -) -> FatalResult<()> -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, -{ +) -> FatalResult<()> { let (background_validation_tx, mut background_validation_rx) = mpsc::channel(16); let mut jobs = HashMap::new(); @@ -179,6 +176,7 @@ where Ok(()) } +#[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] async fn run_iteration( ctx: &mut Context, keystore: SyncCryptoStorePtr, @@ -186,11 +184,7 @@ async fn run_iteration( jobs: &mut HashMap>, background_validation_tx: mpsc::Sender<(Hash, ValidatedCandidateCommand)>, background_validation_rx: &mut mpsc::Receiver<(Hash, ValidatedCandidateCommand)>, -) -> Result<(), Error> -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, -{ +) -> Result<(), Error> { loop { futures::select!( validated_command = background_validation_rx.next().fuse() => { @@ -224,16 +218,13 @@ where } } +#[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] async fn handle_validated_candidate_command( ctx: &mut Context, jobs: &mut HashMap>, relay_parent: Hash, command: ValidatedCandidateCommand, -) -> Result<(), Error> -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, -{ +) -> Result<(), Error> { if let Some(job) = jobs.get_mut(&relay_parent) { job.job.handle_validated_candidate_command(&job.span, ctx, command).await?; } else { @@ -244,15 +235,12 @@ where Ok(()) } +#[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] async fn handle_communication( ctx: &mut Context, jobs: &mut HashMap>, message: CandidateBackingMessage, -) -> Result<(), Error> -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, -{ +) -> Result<(), Error> { match message { CandidateBackingMessage::Second(relay_parent, candidate, pov) => { if let Some(job) = jobs.get_mut(&relay_parent) { @@ -273,6 +261,7 @@ where Ok(()) } +#[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] async fn handle_active_leaves_update( ctx: &mut Context, update: ActiveLeavesUpdate, @@ -280,11 +269,7 @@ async fn handle_active_leaves_update( keystore: &SyncCryptoStorePtr, background_validation_tx: &mpsc::Sender<(Hash, ValidatedCandidateCommand)>, metrics: &Metrics, -) -> Result<(), Error> -where - Context: SubsystemContext, - Context: overseer::SubsystemContext, -{ +) -> Result<(), Error> { for deactivated in update.deactivated { jobs.remove(&deactivated); } @@ -577,22 +562,19 @@ fn table_attested_to_backed( } async fn store_available_data( - ctx: &mut Context, + sender: &mut impl overseer::CandidateBackingSenderTrait, n_validators: u32, candidate_hash: CandidateHash, available_data: AvailableData, ) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); - ctx.sender() - .send_message( - AvailabilityStoreMessage::StoreAvailableData { - candidate_hash, - n_validators, - available_data, - tx, - } - .into(), - ) + sender + .send_message(AvailabilityStoreMessage::StoreAvailableData { + candidate_hash, + n_validators, + available_data, + tx, + }) .await; let _ = rx.await.map_err(Error::StoreAvailableData)?; @@ -604,8 +586,9 @@ async fn store_available_data( // // This will compute the erasure root internally and compare it to the expected erasure root. // This returns `Err()` iff there is an internal error. Otherwise, it returns either `Ok(Ok(()))` or `Ok(Err(_))`. + async fn make_pov_available( - ctx: &mut Context, + sender: &mut impl overseer::CandidateBackingSenderTrait, n_validators: usize, pov: Arc, candidate_hash: CandidateHash, @@ -631,31 +614,28 @@ async fn make_pov_available( { let _span = span.as_ref().map(|s| s.child("store-data").with_candidate(candidate_hash)); - store_available_data(ctx, n_validators as u32, candidate_hash, available_data).await?; + store_available_data(sender, n_validators as u32, candidate_hash, available_data).await?; } Ok(Ok(())) } async fn request_pov( - ctx: &mut Context, + sender: &mut impl overseer::CandidateBackingSenderTrait, relay_parent: Hash, from_validator: ValidatorIndex, candidate_hash: CandidateHash, pov_hash: Hash, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); - ctx.sender() - .send_message( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - from_validator, - candidate_hash, - pov_hash, - tx, - } - .into(), - ) + sender + .send_message(AvailabilityDistributionMessage::FetchPoV { + relay_parent, + from_validator, + candidate_hash, + pov_hash, + tx, + }) .await; let pov = rx.await.map_err(|_| Error::FetchPoV)?; @@ -663,13 +643,13 @@ async fn request_pov( } async fn request_candidate_validation( - ctx: &mut impl overseer::CandidateBackingSenderTrait, + sender: &mut impl overseer::CandidateBackingSenderTrait, candidate_receipt: CandidateReceipt, pov: Arc, ) -> Result { let (tx, rx) = oneshot::channel(); - ctx.sender() + sender .send_message(CandidateValidationMessage::ValidateFromChainState( candidate_receipt, pov, @@ -688,7 +668,7 @@ async fn request_candidate_validation( type BackgroundValidationResult = Result<(CandidateReceipt, CandidateCommitments, Arc), CandidateReceipt>; -struct BackgroundValidationParams, F> { +struct BackgroundValidationParams { sender: S, tx_command: mpsc::Sender<(Hash, ValidatedCandidateCommand)>, candidate: CandidateReceipt, @@ -720,7 +700,8 @@ async fn validate_and_make_available( PoVData::Ready(pov) => pov, PoVData::FetchFromValidator { from_validator, candidate_hash, pov_hash } => { let _span = span.as_ref().map(|s| s.child("request-pov")); - match request_pov(sender, relay_parent, from_validator, candidate_hash, pov_hash).await + match request_pov(&mut sender, relay_parent, from_validator, candidate_hash, pov_hash) + .await { Err(Error::FetchPoV) => { tx_command @@ -804,6 +785,7 @@ async fn validate_and_make_available( struct ValidatorIndexOutOfBounds; +#[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] impl CandidateBackingJob { async fn handle_validated_candidate_command( &mut self, @@ -887,7 +869,7 @@ impl CandidateBackingJob { &mut self, ctx: &mut Context, params: BackgroundValidationParams< - Sender, + impl overseer::CandidateBackingSenderTrait, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync, >, ) -> Result<(), Error> { @@ -992,7 +974,7 @@ impl CandidateBackingJob { } /// Check if there have happened any new misbehaviors and issue necessary messages. - fn issue_new_misbehaviors(&mut self, sender: &mut impl CandidateBackingSenderTrait) { + fn issue_new_misbehaviors(&mut self, sender: &mut impl overseer::CandidateBackingSenderTrait) { // collect the misbehaviors to avoid double mutable self borrow issues let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect(); for (validator_id, report) in misbehaviors { @@ -1033,7 +1015,7 @@ impl CandidateBackingJob { }; if let Err(ValidatorIndexOutOfBounds) = self - .dispatch_new_statement_to_dispute_coordinator(ctx, candidate_hash, &statement) + .dispatch_new_statement_to_dispute_coordinator(ctx.sender(), candidate_hash, &statement) .await { gum::warn!( @@ -1115,7 +1097,7 @@ impl CandidateBackingJob { /// is meant to check the signature and provenance of all statements before submission. async fn dispatch_new_statement_to_dispute_coordinator( &self, - ctx: &mut impl overseer::CandidateBackingContextTrait, + sender: &mut impl overseer::CandidateBackingSenderTrait, candidate_hash: CandidateHash, statement: &SignedFullStatement, ) -> Result<(), ValidatorIndexOutOfBounds> { From ee5db10c2185bd5a28b573455f18bccd8e073379 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 17:12:22 +0200 Subject: [PATCH 093/139] use more ``#[overseer::contextbounds(*, prefix = self::overseer)]`` --- node/core/approval-voting/src/lib.rs | 34 +++------ node/network/approval-distribution/src/lib.rs | 2 +- node/network/bitfield-distribution/src/lib.rs | 20 ++--- .../network/statement-distribution/src/lib.rs | 76 ++++++------------- 4 files changed, 43 insertions(+), 89 deletions(-) diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 7329950faedd..227774b2c019 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -355,10 +355,8 @@ impl ApprovalVotingSubsystem { } } -impl overseer::Subsystem for ApprovalVotingSubsystem -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, +#[overseer::subsystem(ApprovalVoting, error = SubsystemError, prefix = self::overseer)] +impl ApprovalVotingSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); @@ -597,6 +595,7 @@ struct State { assignment_criteria: Box, } +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] impl State { fn session_info(&self, i: SessionIndex) -> Option<&SessionInfo> { self.session_window.as_ref().and_then(|w| w.session_info(i)) @@ -608,10 +607,6 @@ impl State { ctx: &mut Context, head: Hash, ) -> Result, SessionsUnavailable> - where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: - overseer::ApprovalVotingSenderTrait, { let session_window = self.session_window.take(); match session_window { @@ -714,6 +709,7 @@ enum Action { Conclude, } +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn run( mut ctx: Context, mut subsystem: ApprovalVotingSubsystem, @@ -722,8 +718,6 @@ async fn run( mut backend: B, ) -> SubsystemResult<()> where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, B: Backend, { let mut state = State { @@ -861,6 +855,7 @@ where // https://github.com/paritytech/polkadot/issues/3311 // // returns `true` if any of the actions was a `Conclude` command. +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn handle_actions( ctx: &mut Context, state: &mut State, @@ -872,9 +867,6 @@ async fn handle_actions( mode: &mut Mode, actions: Vec, ) -> SubsystemResult -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, { let mut conclude = false; @@ -1101,6 +1093,7 @@ fn distribution_messages_for_activation( } // Handle an incoming signal from the overseer. Returns true if execution should conclude. +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn handle_from_overseer( ctx: &mut Context, state: &mut State, @@ -1110,9 +1103,6 @@ async fn handle_from_overseer( last_finalized_height: &mut Option, wakeups: &mut Wakeups, ) -> SubsystemResult> -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, { let actions = match x { FromOverseer::Signal(OverseerSignal::ActiveLeaves(update)) => { @@ -1215,6 +1205,7 @@ where Ok(actions) } +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn handle_approved_ancestor( ctx: &mut Context, db: &OverlayedBackend<'_, impl Backend>, @@ -1222,9 +1213,6 @@ async fn handle_approved_ancestor( lower_bound: BlockNumber, wakeups: &Wakeups, ) -> SubsystemResult> -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, { const MAX_TRACING_WINDOW: usize = 200; const ABNORMAL_DEPTH_THRESHOLD: usize = 5; @@ -2169,6 +2157,7 @@ fn process_wakeup( // Launch approval work, returning an `AbortHandle` which corresponds to the background task // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn launch_approval( ctx: &mut Context, metrics: Metrics, @@ -2178,9 +2167,6 @@ async fn launch_approval( block_hash: Hash, backing_group: GroupIndex, ) -> SubsystemResult> -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, { let (a_tx, a_rx) = oneshot::channel(); let (code_tx, code_rx) = oneshot::channel(); @@ -2381,6 +2367,7 @@ where // Issue and import a local approval vote. Should only be invoked after approval checks // have been done. +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn issue_approval( ctx: &mut Context, state: &mut State, @@ -2389,9 +2376,6 @@ async fn issue_approval( candidate_hash: CandidateHash, ApprovalVoteRequest { validator_index, block_hash }: ApprovalVoteRequest, ) -> SubsystemResult> -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, { let block_entry = match db.load_block_entry(&block_hash)? { Some(b) => b, diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 338f28ecde56..3619c29b3bbb 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -1731,7 +1731,7 @@ impl ApprovalDistribution { } #[overseer::subsystem(ApprovalDistribution, error=SubsystemError, prefix=self::overseer)] -impl ApprovalDistributionSubsystem { +impl ApprovalDistribution { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx).map(|_| Ok(())).boxed(); diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 95447e872415..ab5296245705 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -318,10 +318,6 @@ impl BitfieldDistribution { /// Modify the reputation of a peer based on its behavior. async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) -where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, { gum::trace!(target: LOG_TARGET, ?relay_parent, ?rep, %peer, "reputation change"); @@ -330,6 +326,7 @@ where /// Distribute a given valid and signature checked bitfield message. /// /// For this variant the source is this node. +#[overseer::contextbounds(BitfieldDistribution, prefix=self::overseer)] async fn handle_bitfield_distribution( ctx: &mut Context, state: &mut ProtocolState, @@ -394,6 +391,7 @@ async fn handle_bitfield_distribution( /// Distribute a given valid and signature checked bitfield message. /// /// Can be originated by another subsystem or received via network from another peer. +#[overseer::contextbounds(BitfieldDistribution, prefix=self::overseer)] async fn relay_message( ctx: &mut Context, job_data: &mut PerRelayParentData, @@ -485,6 +483,7 @@ async fn relay_message( } /// Handle an incoming message from a peer. +#[overseer::contextbounds(BitfieldDistribution, prefix=self::overseer)] async fn process_incoming_peer_message( ctx: &mut Context, state: &mut ProtocolState, @@ -606,6 +605,7 @@ async fn process_incoming_peer_message( /// Deal with network bridge updates and track what needs to be tracked /// which depends on the message type received. +#[overseer::contextbounds(BitfieldDistribution, prefix=self::overseer)] async fn handle_network_msg( ctx: &mut Context, state: &mut ProtocolState, @@ -688,6 +688,7 @@ fn handle_our_view_change(state: &mut ProtocolState, view: OurView) { // Send the difference between two views which were not sent // to that particular peer. +#[overseer::contextbounds(BitfieldDistribution, prefix=self::overseer)] async fn handle_peer_view_change( ctx: &mut Context, state: &mut ProtocolState, @@ -749,16 +750,14 @@ async fn handle_peer_view_change( } /// Send a gossip message and track it in the per relay parent data. +#[overseer::contextbounds(BitfieldDistribution, prefix=self::overseer)] async fn send_tracked_gossip_message( ctx: &mut Context, state: &mut ProtocolState, dest: PeerId, validator: ValidatorId, message: BitfieldGossipMessage, -) where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, +) { let job_data = if let Some(job_data) = state.per_relay_parent.get_mut(&message.relay_parent) { job_data @@ -798,14 +797,11 @@ impl BitfieldDistribution { } /// Query our validator set and signing context for a particular relay parent. +#[overseer::contextbounds(BitfieldDistribution, prefix=self::overseer)] async fn query_basics( ctx: &mut Context, relay_parent: Hash, ) -> SubsystemResult, SigningContext)>> -where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, { let (validators_tx, validators_rx) = oneshot::channel(); let (session_tx, session_rx) = oneshot::channel(); diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 8338a0a2cbc2..e5cab478104d 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -127,12 +127,8 @@ pub struct StatementDistributionSubsystem { rng: R, } -impl overseer::Subsystem - for StatementDistributionSubsystem -where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, +#[overseer::subsystem(StatementDistribution, error= SubsystemError, prefix=self::overseer)] +impl StatementDistributionSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision @@ -895,6 +891,7 @@ fn check_statement_signature( /// circulates the statement to all peers who have not seen it yet, and /// sends all statements dependent on that statement to peers who could previously not receive /// them but now can. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn circulate_statement_and_dependents( gossip_peers: &HashSet, peers: &mut HashMap, @@ -905,10 +902,7 @@ async fn circulate_statement_and_dependents( priority_peers: Vec, metrics: &Metrics, rng: &mut impl rand::Rng, -) where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, +) { let active_head = match active_heads.get_mut(&relay_parent) { Some(res) => res, @@ -1015,6 +1009,7 @@ fn is_statement_large(statement: &SignedFullStatement) -> (bool, Option) /// Circulates a statement to all peers who have not seen it yet, and returns /// an iterator over peers who need to have dependent statements sent. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn circulate_statement<'a, Context>( gossip_peers: &HashSet, peers: &mut HashMap, @@ -1025,10 +1020,7 @@ async fn circulate_statement<'a, Context>( metrics: &Metrics, rng: &mut impl rand::Rng, ) -> Vec -where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, + { let fingerprint = stored.fingerprint(); @@ -1109,6 +1101,7 @@ where } /// Send all statements about a given candidate hash to a peer. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, @@ -1117,10 +1110,7 @@ async fn send_statements_about( candidate_hash: CandidateHash, active_head: &ActiveHeadData, metrics: &Metrics, -) where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, +) { for statement in active_head.statements_about(candidate_hash) { let fingerprint = statement.fingerprint(); @@ -1146,6 +1136,7 @@ async fn send_statements_about( } /// Send all statements at a given relay-parent to a peer. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn send_statements( peer: PeerId, peer_data: &mut PeerData, @@ -1153,10 +1144,7 @@ async fn send_statements( relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, -) where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, +) { for statement in active_head.statements() { let fingerprint = statement.fingerprint(); @@ -1181,10 +1169,7 @@ async fn send_statements( } async fn report_peer(ctx: &mut Context, peer: PeerId, rep: Rep) -where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, + { ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await } @@ -1196,6 +1181,7 @@ where /// your statement. /// /// If the message was large, but the result has been fetched already that one is returned. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn retrieve_statement_from_message<'a, Context>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, @@ -1204,10 +1190,7 @@ async fn retrieve_statement_from_message<'a, Context>( req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option -where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, + { let fingerprint = message.get_fingerprint(); let candidate_hash = *fingerprint.0.candidate_hash(); @@ -1293,6 +1276,7 @@ where /// Launch request for a large statement and get tracking status. /// /// Returns `None` if spawning task failed. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn launch_request( meta: StatementMetadata, peer: PeerId, @@ -1300,10 +1284,7 @@ async fn launch_request( ctx: &mut Context, metrics: &Metrics, ) -> Option -where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, + { let (task, handle) = fetch(meta.relay_parent, meta.candidate_hash, vec![peer], req_sender, metrics.clone()) @@ -1328,7 +1309,7 @@ where } /// Handle incoming message and circulate it to peers, if we did not know it already. -/// +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn handle_incoming_message_and_circulate<'a, Context, R>( peer: PeerId, gossip_peers: &HashSet, @@ -1340,10 +1321,8 @@ async fn handle_incoming_message_and_circulate<'a, Context, R>( req_sender: &mpsc::Sender, metrics: &Metrics, rng: &mut R, -) where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, +) +where R: rand::Rng, { let handled_incoming = match peers.get_mut(&peer) { @@ -1391,6 +1370,7 @@ async fn handle_incoming_message_and_circulate<'a, Context, R>( // // This function checks the signature and ensures the statement is compatible with our // view. It also notifies candidate backing if the statement was previously unknown. +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn handle_incoming_message<'a, Context>( peer: PeerId, peer_data: &mut PeerData, @@ -1401,10 +1381,6 @@ async fn handle_incoming_message<'a, Context>( req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option<(Hash, StoredStatement<'a>)> -where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, { let relay_parent = message.get_relay_parent(); let _ = metrics.time_network_bridge_update_v1("handle_incoming_message"); @@ -1593,6 +1569,7 @@ where } /// Update a peer's view. Sends all newly unlocked statements based on the previous +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn update_peer_view_and_maybe_send_unlocked( peer: PeerId, gossip_peers: &HashSet, @@ -1602,10 +1579,8 @@ async fn update_peer_view_and_maybe_send_unlocked( new_view: View, metrics: &Metrics, rng: &mut R, -) where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, +) +where R: rand::Rng, { let old_view = std::mem::replace(&mut peer_data.view, new_view); @@ -1637,6 +1612,7 @@ async fn update_peer_view_and_maybe_send_unlocked( } } +#[overseer::contextbounds(StatementDistribution, prefix=self::overseer)] async fn handle_network_update( peers: &mut HashMap, gossip_peers: &mut HashSet, @@ -1648,10 +1624,8 @@ async fn handle_network_update( update: NetworkBridgeEvent, metrics: &Metrics, rng: &mut R, -) where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, +) +where R: rand::Rng, { match update { From 8972b69f344f46777c401166f434ab248f7a65de Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 17:39:26 +0200 Subject: [PATCH 094/139] use `#[overseer::contextbounds(..)]` almsot everywhere --- node/core/approval-voting/src/import.rs | 14 +--- node/core/approval-voting/src/lib.rs | 23 ++---- node/core/av-store/src/lib.rs | 40 +++------- node/core/candidate-validation/src/lib.rs | 8 +- node/core/chain-api/src/lib.rs | 8 +- node/core/chain-selection/src/lib.rs | 13 +--- .../dispute-coordinator/src/initialized.rs | 50 ++---------- node/core/dispute-coordinator/src/lib.rs | 30 ++----- .../src/participation/mod.rs | 36 ++------- .../src/participation/tests.rs | 25 ++---- node/core/pvf-checker/src/lib.rs | 14 +--- node/core/runtime-api/src/lib.rs | 8 +- node/network/approval-distribution/src/lib.rs | 78 ++++--------------- .../availability-distribution/src/lib.rs | 8 +- .../src/pov_requester/mod.rs | 8 +- .../src/requester/fetch_task/mod.rs | 8 +- .../src/requester/mod.rs | 22 +----- .../src/requester/session_cache.rs | 10 +-- node/network/availability-recovery/src/lib.rs | 32 ++------ node/network/bitfield-distribution/src/lib.rs | 52 +++---------- node/network/bridge/src/lib.rs | 14 ++-- node/network/gossip-support/src/lib.rs | 13 +--- .../network/statement-distribution/src/lib.rs | 72 +++++------------ node/overseer/examples/minimal-example.rs | 33 ++------ 24 files changed, 145 insertions(+), 474 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index b3ae5d8b4de2..add41c0e4a12 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -107,16 +107,13 @@ enum ImportedBlockInfoError { } /// Computes information about the imported block. Returns an error if the info couldn't be extracted. +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn imported_block_info( ctx: &mut Context, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, -) -> Result -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, -{ +) -> Result { // Ignore any runtime API errors - that means these blocks are old and finalized. // Only unfinalized blocks factor into the approval voting process. @@ -323,17 +320,14 @@ pub struct BlockImportedCandidates { /// * and return information about all candidates imported under each block. /// /// It is the responsibility of the caller to schedule wakeups for each block. +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] pub(crate) async fn handle_new_head( ctx: &mut Context, state: &mut State, db: &mut OverlayedBackend<'_, B>, head: Hash, finalized_number: &Option, -) -> SubsystemResult> -where - Context: overseer::ApprovalVotingContextTrait, - ::Sender: overseer::ApprovalVotingSenderTrait, -{ +) -> SubsystemResult> { const MAX_HEADS_LOOK_BACK: BlockNumber = MAX_FINALITY_LAG; let mut span = jaeger::Span::new(head, "approval-checking-import"); diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 227774b2c019..09df46126da5 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -356,8 +356,7 @@ impl ApprovalVotingSubsystem { } #[overseer::subsystem(ApprovalVoting, error = SubsystemError, prefix = self::overseer)] -impl ApprovalVotingSubsystem -{ +impl ApprovalVotingSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); let future = run::( @@ -606,13 +605,12 @@ impl State { &mut self, ctx: &mut Context, head: Hash, - ) -> Result, SessionsUnavailable> - { + ) -> Result, SessionsUnavailable> { let session_window = self.session_window.take(); match session_window { None => { self.session_window = Some( - RollingSessionWindow::new::( + RollingSessionWindow::new::<::Sender>( ctx.sender().clone(), APPROVAL_SESSIONS, head, @@ -866,8 +864,7 @@ async fn handle_actions( approvals_cache: &mut lru::LruCache, mode: &mut Mode, actions: Vec, -) -> SubsystemResult -{ +) -> SubsystemResult { let mut conclude = false; let mut actions_iter = actions.into_iter(); @@ -1102,8 +1099,7 @@ async fn handle_from_overseer( x: FromOverseer, last_finalized_height: &mut Option, wakeups: &mut Wakeups, -) -> SubsystemResult> -{ +) -> SubsystemResult> { let actions = match x { FromOverseer::Signal(OverseerSignal::ActiveLeaves(update)) => { let mut actions = Vec::new(); @@ -1212,8 +1208,7 @@ async fn handle_approved_ancestor( target: Hash, lower_bound: BlockNumber, wakeups: &Wakeups, -) -> SubsystemResult> -{ +) -> SubsystemResult> { const MAX_TRACING_WINDOW: usize = 200; const ABNORMAL_DEPTH_THRESHOLD: usize = 5; @@ -2166,8 +2161,7 @@ async fn launch_approval( validator_index: ValidatorIndex, block_hash: Hash, backing_group: GroupIndex, -) -> SubsystemResult> -{ +) -> SubsystemResult> { let (a_tx, a_rx) = oneshot::channel(); let (code_tx, code_rx) = oneshot::channel(); @@ -2375,8 +2369,7 @@ async fn issue_approval( metrics: &Metrics, candidate_hash: CandidateHash, ApprovalVoteRequest { validator_index, block_hash }: ApprovalVoteRequest, -) -> SubsystemResult> -{ +) -> SubsystemResult> { let block_entry = match db.load_block_entry(&block_hash)? { Some(b) => b, None => { diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index d443cda6265f..1ae1cf96691c 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -528,12 +528,8 @@ impl AvailabilityStoreSubsystem { } } -async fn run(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context) -where - Context: overseer::AvailabilityStoreContextTrait, - ::Sender: - overseer::AvailabilityStoreSenderTrait, -{ +#[overseer::contextbounds(AvailabilityStore, prefix = self::overseer)] +async fn run(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context) { let mut next_pruning = Delay::new(subsystem.pruning_config.pruning_interval).fuse(); loop { @@ -554,16 +550,12 @@ where } } +#[overseer::contextbounds(AvailabilityStore, prefix = self::overseer)] async fn run_iteration( ctx: &mut Context, subsystem: &mut AvailabilityStoreSubsystem, mut next_pruning: &mut future::Fuse, -) -> Result -where - Context: overseer::AvailabilityStoreContextTrait, - ::Sender: - overseer::AvailabilityStoreSenderTrait, -{ +) -> Result { select! { incoming = ctx.recv().fuse() => { match incoming.map_err(|_| Error::ContextChannelClosed)? { @@ -607,16 +599,12 @@ where Ok(false) } +#[overseer::contextbounds(AvailabilityStore, prefix = self::overseer)] async fn process_block_activated( ctx: &mut Context, subsystem: &mut AvailabilityStoreSubsystem, activated: Hash, -) -> Result<(), Error> -where - Context: overseer::AvailabilityStoreContextTrait, - ::Sender: - overseer::AvailabilityStoreSenderTrait, -{ +) -> Result<(), Error> { let now = subsystem.clock.now()?; let block_header = { @@ -663,6 +651,7 @@ where Ok(()) } +#[overseer::contextbounds(AvailabilityStore, prefix = self::overseer)] async fn process_new_head( ctx: &mut Context, db: &Arc, @@ -672,12 +661,7 @@ async fn process_new_head( now: Duration, hash: Hash, header: Header, -) -> Result<(), Error> -where - Context: overseer::AvailabilityStoreContextTrait, - ::Sender: - overseer::AvailabilityStoreSenderTrait, -{ +) -> Result<(), Error> { let candidate_events = util::request_candidate_events(hash, ctx.sender()).await.await??; // We need to request the number of validators based on the parent state, @@ -815,17 +799,13 @@ macro_rules! peek_num { }; } +#[overseer::contextbounds(AvailabilityStore, prefix = self::overseer)] async fn process_block_finalized( ctx: &mut Context, subsystem: &AvailabilityStoreSubsystem, finalized_hash: Hash, finalized_number: BlockNumber, -) -> Result<(), Error> -where - Context: overseer::AvailabilityStoreContextTrait, - ::Sender: - overseer::AvailabilityStoreSenderTrait, -{ +) -> Result<(), Error> { let now = subsystem.clock.now()?; let mut next_possible_batch = 0; diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 9399f61bc0bb..561977871695 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -109,18 +109,14 @@ impl CandidateValidationSubsystem { } } +#[overseer::contextbounds(CandidateValidation, prefix = self::overseer)] async fn run( mut ctx: Context, metrics: Metrics, pvf_metrics: polkadot_node_core_pvf::Metrics, cache_path: PathBuf, program_path: PathBuf, -) -> SubsystemResult<()> -where - Context: overseer::CandidateValidationContextTrait, - ::Sender: - overseer::CandidateValidationSenderTrait, -{ +) -> SubsystemResult<()> { let (validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), pvf_metrics, diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 4929ae50b4bb..96bdaa1e0b5a 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -64,11 +64,10 @@ impl ChainApiSubsystem { } } -impl overseer::Subsystem for ChainApiSubsystem +#[overseer::subsystem(ChainApi, error = SubsystemError, prefix = self::overseer)] +impl ChainApiSubsystem where Client: HeaderBackend + AuxStore + 'static, - Context: overseer::ChainApiContextTrait, - ::Sender: overseer::ChainApiSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(ctx, self) @@ -78,14 +77,13 @@ where } } +#[overseer::contextbounds(ChainApi, prefix = self::overseer)] async fn run( mut ctx: Context, subsystem: ChainApiSubsystem, ) -> SubsystemResult<()> where Client: HeaderBackend + AuxStore, - Context: overseer::ChainApiContextTrait, - ::Sender: overseer::ChainApiSenderTrait, { loop { match ctx.recv().await? { diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index 1383d3286db7..295437a630da 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -329,11 +329,8 @@ impl ChainSelectionSubsystem { } } -impl overseer::Subsystem for ChainSelectionSubsystem -where - Context: overseer::ChainSelectionContextTrait, - ::Sender: overseer::ChainSelectionSenderTrait, -{ +#[overseer::subsystem(ChainSelection, error = SubsystemError, prefix = self::overseer)] +impl ChainSelectionSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = db_backend::v1::DbBackend::new( self.db, @@ -349,14 +346,13 @@ where } } +#[overseer::contextbounds(ChainSelection, prefix = self::overseer)] async fn run( mut ctx: Context, mut backend: B, stagnant_check_interval: StagnantCheckInterval, clock: Box, ) where - Context: overseer::ChainSelectionContextTrait, - ::Sender: overseer::ChainSelectionSenderTrait, B: Backend, { loop { @@ -380,6 +376,7 @@ async fn run( // // A return value of `Ok` indicates that an exit should be made, while non-fatal errors // lead to another call to this function. +#[overseer::contextbounds(ChainSelection, prefix = self::overseer)] async fn run_until_error( ctx: &mut Context, backend: &mut B, @@ -387,8 +384,6 @@ async fn run_until_error( clock: &(dyn Clock + Sync), ) -> Result<(), Error> where - Context: overseer::ChainSelectionContextTrait, - ::Sender: overseer::ChainSelectionSenderTrait, B: Backend, { let mut stagnant_check_stream = stagnant_check_interval.timeout_stream(); diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index d38e1b5f36f7..c76bcaed10f3 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -83,6 +83,7 @@ pub struct Initialized { error: Option, } +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] impl Initialized { /// Make initialized subsystem, ready to `run`. pub fn new( @@ -123,9 +124,6 @@ impl Initialized { clock: Box, ) -> FatalResult<()> where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, B: Backend, { loop { @@ -162,9 +160,6 @@ impl Initialized { clock: &dyn Clock, ) -> Result<()> where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, B: Backend, { for (priority, request) in participations.drain(..) { @@ -261,12 +256,7 @@ impl Initialized { overlay_db: &mut OverlayedBackend<'_, impl Backend>, update: ActiveLeavesUpdate, now: u64, - ) -> Result<()> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> Result<()> { let on_chain_votes = self.scraper.process_active_leaves_update(ctx.sender(), &update).await?; self.participation.process_active_leaves_update(ctx, &update).await?; @@ -330,12 +320,7 @@ impl Initialized { overlay_db: &mut OverlayedBackend<'_, impl Backend>, votes: ScrapedOnChainVotes, now: u64, - ) -> Result<()> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> Result<()> { let ScrapedOnChainVotes { session, backing_validators_per_candidate, disputes } = votes; if backing_validators_per_candidate.is_empty() && disputes.is_empty() { @@ -513,12 +498,7 @@ impl Initialized { overlay_db: &mut OverlayedBackend<'_, impl Backend>, message: DisputeCoordinatorMessage, now: Timestamp, - ) -> Result JfyiResult<()>>> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> Result JfyiResult<()>>> { match message { DisputeCoordinatorMessage::ImportStatements { candidate_hash, @@ -658,12 +638,7 @@ impl Initialized { session: SessionIndex, statements: Vec<(SignedDisputeStatement, ValidatorIndex)>, now: Timestamp, - ) -> Result - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> Result { if session + DISPUTE_WINDOW.get() < self.highest_session { // It is not valid to participate in an ancient dispute (spam?). return Ok(ImportStatementsResult::InvalidImport) @@ -952,12 +927,7 @@ impl Initialized { session: SessionIndex, valid: bool, now: Timestamp, - ) -> Result<()> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> Result<()> { // Load session info. let info = match self.rolling_session_window.session_info(session) { None => { @@ -1079,16 +1049,12 @@ enum MuxedMessage { Participation(participation::WorkerMessage), } +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] impl MuxedMessage { async fn receive( ctx: &mut Context, from_sender: &mut participation::WorkerMessageReceiver, - ) -> FatalResult - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> FatalResult { // We are only fusing here to make `select` happy, in reality we will quit if the stream // ends. let from_overseer = ctx.recv().fuse(); diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 32c78bfa7447..25b289296b32 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -139,6 +139,7 @@ impl DisputeCoordinatorSubsystem { } } +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] impl DisputeCoordinatorSubsystem { /// Create a new instance of the subsystem. pub fn new( @@ -158,9 +159,6 @@ impl DisputeCoordinatorSubsystem { clock: Box, ) -> FatalResult<()> where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, B: Backend + 'static, { let res = self.initialize(&mut ctx, backend, &*clock).await?; @@ -192,9 +190,6 @@ impl DisputeCoordinatorSubsystem { )>, > where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, B: Backend + 'static, { loop { @@ -259,12 +254,7 @@ impl DisputeCoordinatorSubsystem { Vec, SpamSlots, ChainScraper, - )> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + )> { // Prune obsolete disputes: db::v1::note_current_session(overlay_db, rolling_session_window.latest_session())?; @@ -358,14 +348,10 @@ impl DisputeCoordinatorSubsystem { } /// Wait for `ActiveLeavesUpdate` on startup, returns `None` if `Conclude` signal came first. +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] async fn get_rolling_session_window( ctx: &mut Context, -) -> Result> -where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, -{ +) -> Result> { if let Some(leaf) = wait_for_first_leaf(ctx).await? { Ok(Some(( leaf.clone(), @@ -379,12 +365,8 @@ where } /// Wait for `ActiveLeavesUpdate`, returns `None` if `Conclude` signal came first. -async fn wait_for_first_leaf(ctx: &mut Context) -> Result> -where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, -{ +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] +async fn wait_for_first_leaf(ctx: &mut Context) -> Result> { loop { match ctx.recv().await? { FromOverseer::Signal(OverseerSignal::Conclude) => return Ok(None), diff --git a/node/core/dispute-coordinator/src/participation/mod.rs b/node/core/dispute-coordinator/src/participation/mod.rs index 37008e598126..fd9bb783c651 100644 --- a/node/core/dispute-coordinator/src/participation/mod.rs +++ b/node/core/dispute-coordinator/src/participation/mod.rs @@ -123,6 +123,7 @@ impl WorkerMessage { } } +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] impl Participation { /// Get ready for managing dispute participation requests. /// @@ -149,12 +150,7 @@ impl Participation { ctx: &mut Context, priority: ParticipationPriority, req: ParticipationRequest, - ) -> Result<()> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> Result<()> { // Participation already running - we can ignore that request: if self.running_participations.contains(req.candidate_hash()) { return Ok(()) @@ -183,12 +179,7 @@ impl Participation { &mut self, ctx: &mut Context, msg: WorkerMessage, - ) -> FatalResult - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> FatalResult { let WorkerMessage(statement) = msg; self.running_participations.remove(&statement.candidate_hash); let recent_block = self.recent_block.expect("We never ever reset recent_block to `None` and we already received a result, so it must have been set before. qed."); @@ -204,12 +195,7 @@ impl Participation { &mut self, ctx: &mut Context, update: &ActiveLeavesUpdate, - ) -> FatalResult<()> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> FatalResult<()> { if let Some(activated) = &update.activated { match self.recent_block { None => { @@ -232,12 +218,7 @@ impl Participation { &mut self, ctx: &mut Context, recent_head: Hash, - ) -> FatalResult<()> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> FatalResult<()> { while self.running_participations.len() < MAX_PARALLEL_PARTICIPATIONS { if let Some(req) = self.queue.dequeue() { self.fork_participation(ctx, req, recent_head)?; @@ -254,12 +235,7 @@ impl Participation { ctx: &mut Context, req: ParticipationRequest, recent_head: Hash, - ) -> FatalResult<()> - where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, - { + ) -> FatalResult<()> { if self.running_participations.insert(req.candidate_hash().clone()) { let sender = ctx.sender().clone(); ctx.spawn( diff --git a/node/core/dispute-coordinator/src/participation/tests.rs b/node/core/dispute-coordinator/src/participation/tests.rs index 68962af2023f..37400f365fb1 100644 --- a/node/core/dispute-coordinator/src/participation/tests.rs +++ b/node/core/dispute-coordinator/src/participation/tests.rs @@ -52,26 +52,19 @@ pub fn make_our_subsystem_context( ) { make_subsystem_context(spawn) } -async fn participate(ctx: &mut Context, participation: &mut Participation) -> Result<()> -where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, -{ + +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] +async fn participate(ctx: &mut Context, participation: &mut Participation) -> Result<()> { let commitments = CandidateCommitments::default(); participate_with_commitments_hash(ctx, participation, commitments.hash()).await } +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] async fn participate_with_commitments_hash( ctx: &mut Context, participation: &mut Participation, commitments_hash: Hash, -) -> Result<()> -where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, -{ +) -> Result<()> { let candidate_receipt = { let mut receipt = dummy_candidate_receipt_bad_sig(dummy_hash(), dummy_hash()); receipt.commitments_hash = commitments_hash; @@ -87,16 +80,12 @@ where .await } +#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)] async fn activate_leaf( ctx: &mut Context, participation: &mut Participation, block_number: BlockNumber, -) -> FatalResult<()> -where - Context: overseer::DisputeCoordinatorContextTrait, - ::Sender: - overseer::DisputeCoordinatorSenderTrait, -{ +) -> FatalResult<()> { let block_header = Header { parent_hash: BlakeTwo256::hash(&block_number.encode()), number: block_number, diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index 5b4bf2f00c0e..e9eef19006eb 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -60,11 +60,8 @@ impl PvfCheckerSubsystem { } } -impl overseer::Subsystem for PvfCheckerSubsystem -where - Context: overseer::PvfCheckerContextTrait, - ::Sender: overseer::PvfCheckerSenderTrait, -{ +#[overseer::subsystem(PvfChecker, error=SubsystemError, prefix = self::overseer)] +impl PvfCheckerSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { if self.enabled { let future = run(ctx, self.keystore, self.metrics) @@ -123,15 +120,12 @@ struct State { FuturesUnordered>>, } +#[overseer::contextbounds(PvfChecker, prefix = self::overseer)] async fn run( mut ctx: Context, keystore: SyncCryptoStorePtr, metrics: Metrics, -) -> SubsystemResult<()> -where - Context: overseer::PvfCheckerContextTrait, - ::Sender: overseer::PvfCheckerSenderTrait, -{ +) -> SubsystemResult<()> { let mut state = State { credentials: None, recent_block: None, diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 3275273db940..dfdcc1698efc 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -92,12 +92,11 @@ impl RuntimeApiSubsystem { } } -impl overseer::Subsystem for RuntimeApiSubsystem +#[overseer::subsystem(RuntimeApi, error = SubsystemError, prefix = self::overseer)] +impl RuntimeApiSubsystem where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: overseer::RuntimeApiContextTrait, - ::Sender: overseer::RuntimeApiSenderTrait, { fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { future: run(ctx, self).boxed(), name: "runtime-api-subsystem" } @@ -333,6 +332,7 @@ where } } +#[overseer::subsystem(RuntimeApi, error = SubsystemError, prefix = self::overseer)] async fn run( mut ctx: Context, mut subsystem: RuntimeApiSubsystem, @@ -340,8 +340,6 @@ async fn run( where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: overseer::RuntimeApiContextTrait, - ::Sender: overseer::RuntimeApiSenderTrait, { loop { select! { diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 3619c29b3bbb..8985d48afbac 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -321,6 +321,7 @@ enum PendingMessage { Approval(IndirectSignedApprovalVote), } +#[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] impl State { async fn handle_network_msg( &mut self, @@ -328,11 +329,7 @@ impl State { metrics: &Metrics, event: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { match event { NetworkBridgeEvent::PeerConnected(peer_id, role, _, _) => { // insert a blank view if none already present @@ -386,11 +383,7 @@ impl State { metrics: &Metrics, metas: Vec, rng: &mut (impl CryptoRng + Rng), - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { let mut new_hashes = HashSet::new(); for meta in &metas { match self.blocks.entry(meta.hash.clone()) { @@ -508,11 +501,7 @@ impl State { ctx: &mut Context, session: SessionIndex, topology: SessionGridTopology, - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { self.topologies.insert_topology(session, topology); let topology = self.topologies.get_topology(session).expect("just inserted above; qed"); @@ -538,9 +527,6 @@ impl State { msg: protocol_v1::ApprovalDistributionMessage, rng: &mut R, ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, R: CryptoRng + Rng, { match msg { @@ -634,9 +620,6 @@ impl State { view: View, rng: &mut R, ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, R: CryptoRng + Rng, { gum::trace!(target: LOG_TARGET, ?view, "Peer view change"); @@ -681,11 +664,7 @@ impl State { ctx: &mut Context, metrics: &Metrics, finalized_number: BlockNumber, - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { // we want to prune every block up to (including) finalized_number // why +1 here? // split_off returns everything after the given key, including the key @@ -717,9 +696,6 @@ impl State { claimed_candidate_index: CandidateIndex, rng: &mut R, ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, R: CryptoRng + Rng, { let block_hash = assignment.block_hash.clone(); @@ -977,11 +953,7 @@ impl State { metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { let block_hash = vote.block_hash.clone(); let validator_index = vote.validator; let candidate_index = vote.candidate_index; @@ -1387,11 +1359,7 @@ impl State { ctx: &mut Context, resend: Resend, metrics: &Metrics, - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { let min_age = self.blocks_by_number.iter().next().map(|(num, _)| num); let max_age = self.blocks_by_number.iter().rev().next().map(|(num, _)| num); let config = self.aggression_config.clone(); @@ -1489,6 +1457,7 @@ impl State { // // Note that the required routing of a message can be modified even if the // topology is unknown yet. +#[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] async fn adjust_required_routing_and_propagate( ctx: &mut Context, blocks: &mut HashMap, @@ -1496,9 +1465,6 @@ async fn adjust_required_routing_and_propagate::Sender: - overseer::ApprovalDistributionSenderTrait, BlockFilter: Fn(&mut BlockEntry) -> bool, RoutingModifier: Fn(&mut RequiredRouting, bool, &ValidatorIndex), { @@ -1602,12 +1568,8 @@ async fn adjust_required_routing_and_propagate(ctx: &mut Context, peer_id: PeerId, rep: Rep) -where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, -{ +#[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] +async fn modify_reputation(ctx: &mut Context, peer_id: PeerId, rep: Rep) { gum::trace!( target: LOG_TARGET, reputation = ?rep, @@ -1618,18 +1580,14 @@ where ctx.send_message(NetworkBridgeMessage::ReportPeer(peer_id, rep)).await; } +#[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] impl ApprovalDistribution { /// Create a new instance of the [`ApprovalDistribution`] subsystem. pub fn new(metrics: Metrics) -> Self { Self { metrics } } - async fn run(self, ctx: Context) - where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + async fn run(self, ctx: Context) { let mut state = State::default(); // According to the docs of `rand`, this is a ChaCha12 RNG in practice @@ -1644,11 +1602,7 @@ impl ApprovalDistribution { mut ctx: Context, state: &mut State, rng: &mut (impl CryptoRng + Rng), - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { loop { let message = match ctx.recv().await { Ok(message) => message, @@ -1683,11 +1637,7 @@ impl ApprovalDistribution { msg: ApprovalDistributionMessage, metrics: &Metrics, rng: &mut (impl CryptoRng + Rng), - ) where - Context: overseer::ApprovalDistributionContextTrait, - ::Sender: - overseer::ApprovalDistributionSenderTrait, - { + ) { match msg { ApprovalDistributionMessage::NetworkBridgeUpdate(event) => { state.handle_network_msg(ctx, metrics, event, rng).await; diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index ae379cbb330a..d04b9cbd4bdb 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -80,6 +80,7 @@ impl AvailabilityDistributionSubsystem { } } +#[overseer::contextbounds(AvailabilityDistribution, prefix = self::overseer)] impl AvailabilityDistributionSubsystem { /// Create a new instance of the availability distribution. pub fn new( @@ -92,12 +93,7 @@ impl AvailabilityDistributionSubsystem { } /// Start processing work as passed on from the Overseer. - async fn run(self, mut ctx: Context) -> std::result::Result<(), FatalError> - where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, - { + async fn run(self, mut ctx: Context) -> std::result::Result<(), FatalError> { let Self { mut runtime, recvs, metrics } = self; let IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver } = recvs; diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index ce9e755fba11..0398ca2d267b 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -39,6 +39,7 @@ use crate::{ }; /// Start background worker for taking care of fetching the requested `PoV` from the network. +#[overseer::contextbounds(AvailabilityDistribution, prefix = self::overseer)] pub async fn fetch_pov( ctx: &mut Context, runtime: &mut RuntimeInfo, @@ -48,12 +49,7 @@ pub async fn fetch_pov( pov_hash: Hash, tx: oneshot::Sender, metrics: Metrics, -) -> Result<()> -where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, -{ +) -> Result<()> { let info = &runtime.get_session_info(ctx.sender(), parent).await?.session_info; let authority_id = info .discovery_keys diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index 0f092151f25c..1a133df9cdd0 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -171,16 +171,12 @@ impl FetchTaskConfig { } } +#[overseer::contextbounds(AvailabilityDistribution, prefix = self::overseer)] impl FetchTask { /// Start fetching a chunk. /// /// A task handling the fetching of the configured chunk will be spawned. - pub async fn start(config: FetchTaskConfig, ctx: &mut Context) -> Result - where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, - { + pub async fn start(config: FetchTaskConfig, ctx: &mut Context) -> Result { let FetchTaskConfig { prepared_running, live_in } = config; if let Some(running) = prepared_running { diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index 1c511647b2fe..7b87921d8bb2 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -78,6 +78,7 @@ pub struct Requester { metrics: Metrics, } +#[overseer::contextbounds(AvailabilityDistribution, prefix = self::overseer)] impl Requester { /// How many ancestors of the leaf should we consider along with it. pub(crate) const LEAF_ANCESTRY_LEN_WITHIN_SESSION: usize = 3; @@ -99,12 +100,7 @@ impl Requester { ctx: &mut Context, runtime: &mut RuntimeInfo, update: ActiveLeavesUpdate, - ) -> Result<()> - where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, - { + ) -> Result<()> { gum::trace!(target: LOG_TARGET, ?update, "Update fetching heads"); let ActiveLeavesUpdate { activated, deactivated } = update; // Stale leaves happen after a reversion - we don't want to re-run availability there. @@ -127,12 +123,7 @@ impl Requester { ctx: &mut Context, runtime: &mut RuntimeInfo, new_head: ActivatedLeaf, - ) -> Result<()> - where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, - { + ) -> Result<()> { let sender = &mut ctx.sender().clone(); let ActivatedLeaf { hash: leaf, .. } = new_head; let (leaf_session_index, ancestors_in_session) = get_block_ancestors_in_same_session( @@ -187,12 +178,7 @@ impl Requester { leaf: Hash, leaf_session_index: SessionIndex, cores: impl IntoIterator, - ) -> Result<()> - where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, - { + ) -> Result<()> { for core in cores { match self.fetches.entry(core.candidate_hash) { Entry::Occupied(mut e) => diff --git a/node/network/availability-distribution/src/requester/session_cache.rs b/node/network/availability-distribution/src/requester/session_cache.rs index 676a90b3ce4a..60a7aaf1b345 100644 --- a/node/network/availability-distribution/src/requester/session_cache.rs +++ b/node/network/availability-distribution/src/requester/session_cache.rs @@ -103,9 +103,6 @@ impl SessionCache { with_info: F, ) -> Result> where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, F: FnOnce(&SessionInfo) -> R, { if let Some(o_info) = self.session_info_cache.get(&session_index) { @@ -180,12 +177,7 @@ impl SessionCache { runtime: &mut RuntimeInfo, relay_parent: Hash, session_index: SessionIndex, - ) -> Result> - where - Context: overseer::AvailabilityDistributionContextTrait, - ::Sender: - overseer::AvailabilityDistributionSenderTrait, - { + ) -> Result> { let info = runtime .get_session_info_by_index(ctx.sender(), relay_parent, session_index) .await?; diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 67b3dab30fd7..7a17ff4f0669 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -836,6 +836,7 @@ async fn handle_signal(state: &mut State, signal: OverseerSignal) -> SubsystemRe } /// Machinery around launching recovery tasks into the background. +#[overseer::contextbounds(AvailabilityRecovery, prefix = self::overseer)] async fn launch_recovery_task( state: &mut State, ctx: &mut Context, @@ -844,12 +845,7 @@ async fn launch_recovery_task( backing_group: Option, response_sender: oneshot::Sender>, metrics: &Metrics, -) -> error::Result<()> -where - Context: overseer::AvailabilityRecoveryContextTrait, - ::Sender: - overseer::AvailabilityRecoverySenderTrait, -{ +) -> error::Result<()> { let candidate_hash = receipt.hash(); let params = RecoveryParams { @@ -890,6 +886,7 @@ where } /// Handles an availability recovery request. +#[overseer::contextbounds(AvailabilityRecovery, prefix = self::overseer)] async fn handle_recover( state: &mut State, ctx: &mut Context, @@ -898,12 +895,7 @@ async fn handle_recover( backing_group: Option, response_sender: oneshot::Sender>, metrics: &Metrics, -) -> error::Result<()> -where - Context: overseer::AvailabilityRecoveryContextTrait, - ::Sender: - overseer::AvailabilityRecoverySenderTrait, -{ +) -> error::Result<()> { let candidate_hash = receipt.hash(); let span = jaeger::Span::new(candidate_hash, "availbility-recovery") @@ -959,15 +951,11 @@ where } /// Queries a chunk from av-store. +#[overseer::contextbounds(AvailabilityRecovery, prefix = self::overseer)] async fn query_full_data( ctx: &mut Context, candidate_hash: CandidateHash, -) -> error::Result> -where - Context: overseer::AvailabilityRecoveryContextTrait, - ::Sender: - overseer::AvailabilityRecoverySenderTrait, -{ +) -> error::Result> { let (tx, rx) = oneshot::channel(); ctx.send_message(AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx)) .await; @@ -975,6 +963,7 @@ where Ok(rx.await.map_err(error::Error::CanceledQueryFullData)?) } +#[overseer::contextbounds(AvailabilityRecovery, prefix = self::overseer)] impl AvailabilityRecoverySubsystem { /// Create a new instance of `AvailabilityRecoverySubsystem` which starts with a fast path to /// request data from backers. @@ -993,12 +982,7 @@ impl AvailabilityRecoverySubsystem { Self { fast_path: false, req_receiver, metrics } } - async fn run(self, mut ctx: Context) -> SubsystemResult<()> - where - Context: overseer::AvailabilityRecoveryContextTrait, - ::Sender: - overseer::AvailabilityRecoverySenderTrait, - { + async fn run(self, mut ctx: Context) -> SubsystemResult<()> { let mut state = State::default(); let Self { fast_path, mut req_receiver, metrics } = self; diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index ab5296245705..0745fa4c221c 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -204,6 +204,7 @@ pub struct BitfieldDistribution { metrics: Metrics, } +#[overseer::contextbounds(BitfieldDistribution, prefix = self::overseer)] impl BitfieldDistribution { /// Create a new instance of the `BitfieldDistribution` subsystem. pub fn new(metrics: Metrics) -> Self { @@ -211,12 +212,7 @@ impl BitfieldDistribution { } /// Start processing work as passed on from the Overseer. - async fn run(self, ctx: Context) - where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, - { + async fn run(self, ctx: Context) { let mut state = ProtocolState::default(); let mut rng = rand::rngs::StdRng::from_entropy(); self.run_inner(ctx, &mut state, &mut rng).await @@ -227,10 +223,7 @@ impl BitfieldDistribution { mut ctx: Context, state: &mut ProtocolState, rng: &mut (impl CryptoRng + Rng), - ) where - Context: SubsystemContext, - Context: overseer::SubsystemContext, - { + ) { // work: process incoming messages from the overseer and process accordingly. loop { @@ -317,8 +310,7 @@ impl BitfieldDistribution { } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) -{ +async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) { gum::trace!(target: LOG_TARGET, ?relay_parent, ?rep, %peer, "reputation change"); ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await @@ -334,11 +326,7 @@ async fn handle_bitfield_distribution( relay_parent: Hash, signed_availability: SignedAvailabilityBitfield, rng: &mut (impl CryptoRng + Rng), -) where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, -{ +) { let _timer = metrics.time_handle_bitfield_distribution(); // Ignore anything the overseer did not tell this subsystem to work on @@ -401,11 +389,7 @@ async fn relay_message( message: BitfieldGossipMessage, required_routing: RequiredRouting, rng: &mut (impl CryptoRng + Rng), -) where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, -{ +) { let relay_parent = message.relay_parent; let span = job_data.span.child("relay-msg"); @@ -491,11 +475,7 @@ async fn process_incoming_peer_message( origin: PeerId, message: protocol_v1::BitfieldDistributionMessage, rng: &mut (impl CryptoRng + Rng), -) where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, -{ +) { let protocol_v1::BitfieldDistributionMessage::Bitfield(relay_parent, bitfield) = message; gum::trace!( target: LOG_TARGET, @@ -612,11 +592,7 @@ async fn handle_network_msg( metrics: &Metrics, bridge_message: NetworkBridgeEvent, rng: &mut (impl CryptoRng + Rng), -) where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, -{ +) { let _timer = metrics.time_handle_network_msg(); match bridge_message { @@ -695,11 +671,7 @@ async fn handle_peer_view_change( origin: PeerId, view: View, rng: &mut (impl CryptoRng + Rng), -) where - Context: overseer::BitfieldDistributionContextTrait, - ::Sender: - overseer::BitfieldDistributionSenderTrait, -{ +) { let added = state .peer_views .entry(origin.clone()) @@ -757,8 +729,7 @@ async fn send_tracked_gossip_message( dest: PeerId, validator: ValidatorId, message: BitfieldGossipMessage, -) -{ +) { let job_data = if let Some(job_data) = state.per_relay_parent.get_mut(&message.relay_parent) { job_data } else { @@ -801,8 +772,7 @@ impl BitfieldDistribution { async fn query_basics( ctx: &mut Context, relay_parent: Hash, -) -> SubsystemResult, SigningContext)>> -{ +) -> SubsystemResult, SigningContext)>> { let (validators_tx, validators_rx) = oneshot::channel(); let (session_tx, session_rx) = oneshot::channel(); diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 14921059c011..3fd59596e41a 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -126,12 +126,11 @@ impl NetworkBridge { } } -impl Subsystem for NetworkBridge +#[overseer::subsystem(NetworkBridge, error = SubsystemError, prefix = self::overseer)] +impl NetworkBridge where Net: Network + Sync, AD: validator_discovery::AuthorityDiscovery + Clone, - Context: overseer::NetworkBridgeContextTrait, - ::Sender: overseer::NetworkBridgeSenderTrait, { fn start(mut self, ctx: Context) -> SpawnedSubsystem { // The stream of networking events has to be created at initialization, otherwise the @@ -188,6 +187,7 @@ enum Mode { Active, } +#[overseer::contextbounds(NetworkBridge, prefix = self::overseer)] async fn handle_subsystem_messages( mut ctx: Context, mut network_service: N, @@ -197,8 +197,6 @@ async fn handle_subsystem_messages( metrics: Metrics, ) -> Result<(), UnexpectedAbort> where - Context: overseer::NetworkBridgeContextTrait, - ::Sender: overseer::NetworkBridgeSenderTrait, N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, { @@ -837,6 +835,7 @@ async fn handle_network_messages( /// #fn is_send(); /// #is_send::(); /// ``` +#[overseer::contextbounds(NetworkBridge, prefix = self::overseer)] async fn run_network( bridge: NetworkBridge, mut ctx: Context, @@ -845,8 +844,6 @@ async fn run_network( where N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, - Context: overseer::NetworkBridgeContextTrait, - ::Sender: overseer::NetworkBridgeSenderTrait, { let shared = Shared::default(); @@ -911,6 +908,7 @@ fn construct_view( View::new(live_heads.take(MAX_VIEW_HEADS), finalized_number) } +#[overseer::contextbounds(NetworkBridge, prefix = self::overseer)] fn update_our_view( net: &mut Net, ctx: &mut Context, @@ -920,8 +918,6 @@ fn update_our_view( metrics: &Metrics, ) where Net: Network, - Context: overseer::NetworkBridgeContextTrait, - ::Sender: overseer::NetworkBridgeSenderTrait, { let new_view = construct_view(live_heads.iter().map(|v| v.hash), finalized_number); diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 9a882f07ab34..0d026cf32bb4 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -115,6 +115,7 @@ pub struct GossipSupport { metrics: Metrics, } +#[overseer::contextbounds(GossipSupport, prefix = self::overseer)] impl GossipSupport where AD: AuthorityDiscovery, @@ -138,12 +139,7 @@ where } } - async fn run(mut self, mut ctx: Context) -> Self - where - Context: overseer::GossipSupportContextTrait, - ::Sender: - overseer::GossipSupportSenderTrait, - { + async fn run(mut self, mut ctx: Context) -> Self { fn get_connectivity_check_delay() -> Delay { Delay::new(LOW_CONNECTIVITY_WARN_DELAY) } @@ -586,10 +582,9 @@ fn matrix_neighbors( } } -impl overseer::Subsystem for GossipSupport +#[overseer::subsystem(GossipSupport, error = SubsystemError, prefix = self::overseer)] +impl GossipSupport where - Context: overseer::GossipSupportContextTrait, - ::Sender: overseer::GossipSupportSenderTrait, AD: AuthorityDiscovery + Clone, { fn start(self, ctx: Context) -> SpawnedSubsystem { diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index e5cab478104d..0ee5eaac5e7c 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -127,9 +127,8 @@ pub struct StatementDistributionSubsystem { rng: R, } -#[overseer::subsystem(StatementDistribution, error= SubsystemError, prefix=self::overseer)] -impl StatementDistributionSubsystem -{ +#[overseer::subsystem(StatementDistribution, error=SubsystemError, prefix=self::overseer)] +impl StatementDistributionSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision // within `run`. @@ -627,17 +626,13 @@ enum MuxedMessage { Responder(Option), } +#[overseer::contextbounds(StatementDistribution, prefix = self::overseer)] impl MuxedMessage { async fn receive( ctx: &mut Context, from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, - ) -> MuxedMessage - where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, - { + ) -> MuxedMessage { // We are only fusing here to make `select` happy, in reality we will quit if one of those // streams end: let from_overseer = ctx.recv().fuse(); @@ -902,8 +897,7 @@ async fn circulate_statement_and_dependents( priority_peers: Vec, metrics: &Metrics, rng: &mut impl rand::Rng, -) -{ +) { let active_head = match active_heads.get_mut(&relay_parent) { Some(res) => res, None => return, @@ -1019,9 +1013,7 @@ async fn circulate_statement<'a, Context>( mut priority_peers: Vec, metrics: &Metrics, rng: &mut impl rand::Rng, -) -> Vec - -{ +) -> Vec { let fingerprint = stored.fingerprint(); let mut peers_to_send: Vec = peers @@ -1110,8 +1102,7 @@ async fn send_statements_about( candidate_hash: CandidateHash, active_head: &ActiveHeadData, metrics: &Metrics, -) -{ +) { for statement in active_head.statements_about(candidate_hash) { let fingerprint = statement.fingerprint(); if !peer_data.can_send(&relay_parent, &fingerprint) { @@ -1144,8 +1135,7 @@ async fn send_statements( relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, -) -{ +) { for statement in active_head.statements() { let fingerprint = statement.fingerprint(); if !peer_data.can_send(&relay_parent, &fingerprint) { @@ -1168,9 +1158,7 @@ async fn send_statements( } } -async fn report_peer(ctx: &mut Context, peer: PeerId, rep: Rep) - -{ +async fn report_peer(ctx: &mut Context, peer: PeerId, rep: Rep) { ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await } @@ -1189,9 +1177,7 @@ async fn retrieve_statement_from_message<'a, Context>( ctx: &mut Context, req_sender: &mpsc::Sender, metrics: &Metrics, -) -> Option - -{ +) -> Option { let fingerprint = message.get_fingerprint(); let candidate_hash = *fingerprint.0.candidate_hash(); @@ -1283,9 +1269,7 @@ async fn launch_request( req_sender: mpsc::Sender, ctx: &mut Context, metrics: &Metrics, -) -> Option - -{ +) -> Option { let (task, handle) = fetch(meta.relay_parent, meta.candidate_hash, vec![peer], req_sender, metrics.clone()) .remote_handle(); @@ -1321,8 +1305,7 @@ async fn handle_incoming_message_and_circulate<'a, Context, R>( req_sender: &mpsc::Sender, metrics: &Metrics, rng: &mut R, -) -where +) where R: rand::Rng, { let handled_incoming = match peers.get_mut(&peer) { @@ -1380,8 +1363,7 @@ async fn handle_incoming_message<'a, Context>( message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, -) -> Option<(Hash, StoredStatement<'a>)> -{ +) -> Option<(Hash, StoredStatement<'a>)> { let relay_parent = message.get_relay_parent(); let _ = metrics.time_network_bridge_update_v1("handle_incoming_message"); @@ -1579,8 +1561,7 @@ async fn update_peer_view_and_maybe_send_unlocked( new_view: View, metrics: &Metrics, rng: &mut R, -) -where +) where R: rand::Rng, { let old_view = std::mem::replace(&mut peer_data.view, new_view); @@ -1624,8 +1605,7 @@ async fn handle_network_update( update: NetworkBridgeEvent, metrics: &Metrics, rng: &mut R, -) -where +) where R: rand::Rng, { match update { @@ -1721,6 +1701,7 @@ where } } +#[overseer::contextbounds(StatementDistribution, prefix = self::overseer)] impl StatementDistributionSubsystem { /// Create a new Statement Distribution Subsystem pub fn new( @@ -1732,12 +1713,7 @@ impl StatementDistributionSubsystem { Self { keystore, req_receiver: Some(req_receiver), metrics, rng } } - async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> - where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, - { + async fn run(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> { let mut peers: HashMap = HashMap::new(); let mut gossip_peers: HashSet = HashSet::new(); let mut authorities: HashMap = HashMap::new(); @@ -1862,12 +1838,7 @@ impl StatementDistributionSubsystem { recent_outdated_heads: &RecentOutdatedHeads, req_sender: &mpsc::Sender, message: RequesterMessage, - ) -> JfyiErrorResult<()> - where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, - { + ) -> JfyiErrorResult<()> { match message { RequesterMessage::Finished { relay_parent, @@ -1977,12 +1948,7 @@ impl StatementDistributionSubsystem { recent_outdated_heads: &mut RecentOutdatedHeads, req_sender: &mpsc::Sender, message: FromOverseer, - ) -> Result - where - Context: overseer::StatementDistributionContextTrait, - ::Sender: - overseer::StatementDistributionSenderTrait, - { + ) -> Result { let metrics = &self.metrics; match message { diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index e46f9e5e2850..28c66d57ece7 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -46,12 +46,9 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { struct Subsystem1; +#[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] impl Subsystem1 { - async fn run(mut ctx: Ctx) -> () - where - Ctx: overseer::SubsystemContext, - ::OutgoingMessages: From, - { + async fn run(mut ctx: Context) { 'louy: loop { match ctx.try_recv().await { Ok(Some(msg)) => { @@ -87,14 +84,8 @@ impl Subsystem1 { } } -impl overseer::Subsystem for Subsystem1 -where - Context: overseer::SubsystemContext< - Message = CandidateBackingMessage, - OutgoingMessages = overseer::CandidateBackingOutgoingMessages, - Signal = OverseerSignal, - >, -{ +#[overseer::contextbounds(CandidateBacking, error = SubsystemError, prefix = self::overseer)] +impl Subsystem1 { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { Self::run(ctx).await; @@ -109,14 +100,9 @@ where struct Subsystem2; +#[overseer::contextbounds(CandidateValidation, prefix = self::overseer)] impl Subsystem2 { - async fn run(mut ctx: Ctx) - where - Ctx: overseer::SubsystemContext< - Message = CandidateValidationMessage, - Signal = OverseerSignal, - >, - { + async fn run(mut ctx: Context) -> () { ctx.spawn( "subsystem-2-job", Box::pin(async { @@ -146,11 +132,8 @@ impl Subsystem2 { } } -impl overseer::Subsystem for Subsystem2 -where - Context: - overseer::SubsystemContext, -{ +#[overseer::subsystem(CandidateValidation, error = SubsystemError, prefix = self::overseer)] +impl Subsystem2 { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { Self::run(ctx).await; From b3192ae61a080efe5d3db6bc37128f07058f2241 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 4 May 2022 19:59:37 +0200 Subject: [PATCH 095/139] polkadot compiles now but Finished dev [unoptimized + debuginfo] target(s) in 4m 09s --- Cargo.lock | 1 + node/collation-generation/src/lib.rs | 14 +- node/core/approval-voting/src/lib.rs | 12 +- node/core/dispute-coordinator/src/lib.rs | 7 +- node/core/runtime-api/src/lib.rs | 2 +- node/network/approval-distribution/src/lib.rs | 35 ++-- .../src/requester/session_cache.rs | 1 + node/network/bitfield-distribution/src/lib.rs | 20 +-- node/network/bridge/src/lib.rs | 52 +++--- .../src/collator_side/mod.rs | 57 ++----- node/network/collator-protocol/src/lib.rs | 13 +- .../src/validator_side/mod.rs | 153 +++++++----------- .../network/statement-distribution/src/lib.rs | 28 ++-- node/overseer/src/lib.rs | 4 +- node/service/Cargo.toml | 1 + node/service/src/overseer.rs | 8 +- 16 files changed, 162 insertions(+), 246 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38675f9ceec9..b106534ac7f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7401,6 +7401,7 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", "polkadot-overseer", "polkadot-parachain", diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index c3b7115fc7cc..be901037d33e 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -54,6 +54,7 @@ pub struct CollationGenerationSubsystem { metrics: Metrics, } +#[overseer::contextbounds(CollationGeneration, prefix = self::overseer)] impl CollationGenerationSubsystem { /// Create a new instance of the `CollationGenerationSubsystem`. pub fn new(metrics: Metrics) -> Self { @@ -72,10 +73,6 @@ impl CollationGenerationSubsystem { /// If `err_tx` is not `None`, errors are forwarded onto that channel as they occur. /// Otherwise, most are logged and then discarded. async fn run(mut self, mut ctx: Context) - where - Context: overseer::CollationGenerationContextTrait, - ::Sender: - overseer::CollationGenerationSenderTrait, { // when we activate new leaves, we spawn a bunch of sub-tasks, each of which is // expected to generate precisely one message. We don't want to block the main loop @@ -111,10 +108,6 @@ impl CollationGenerationSubsystem { ctx: &mut Context, sender: &mpsc::Sender, ) -> bool - where - Context: overseer::CollationGenerationContextTrait, - ::Sender: - overseer::CollationGenerationSenderTrait, { match incoming { Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { @@ -177,6 +170,7 @@ impl CollationGenerationSubsystem { } } +#[overseer::contextbounds(CollationGeneration, prefix = self::overseer)] async fn handle_new_activations( config: Arc, activated: impl IntoIterator, @@ -184,10 +178,6 @@ async fn handle_new_activations( metrics: Metrics, sender: &mpsc::Sender, ) -> crate::error::Result<()> -where - Context: overseer::CollationGenerationContextTrait, - ::Sender: - overseer::CollationGenerationSenderTrait, { // follow the procedure from the guide: // https://w3f.github.io/parachain-implementers-guide/node/collators/collation-generation.html diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 09df46126da5..7da70a82ecda 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -356,7 +356,7 @@ impl ApprovalVotingSubsystem { } #[overseer::subsystem(ApprovalVoting, error = SubsystemError, prefix = self::overseer)] -impl ApprovalVotingSubsystem { +impl ApprovalVotingSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = DbBackend::new(self.db.clone(), self.db_config); let future = run::( @@ -605,13 +605,17 @@ impl State { &mut self, ctx: &mut Context, head: Hash, - ) -> Result, SessionsUnavailable> { + ) -> Result, SessionsUnavailable> + where + ::Sender: Sized + Send, + { let session_window = self.session_window.take(); match session_window { None => { + let sender = ctx.sender().clone(); self.session_window = Some( - RollingSessionWindow::new::<::Sender>( - ctx.sender().clone(), + RollingSessionWindow::new( + sender, APPROVAL_SESSIONS, head, ) diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 25b289296b32..f771f0b11e6c 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -125,7 +125,7 @@ impl Config { } #[overseer::subsystem(DisputeCoordinator, error=SubsystemError, prefix=self::overseer)] -impl DisputeCoordinatorSubsystem { +impl DisputeCoordinatorSubsystem { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async { let backend = DbBackend::new(self.store.clone(), self.config.column_config()); @@ -352,10 +352,11 @@ impl DisputeCoordinatorSubsystem { async fn get_rolling_session_window( ctx: &mut Context, ) -> Result> { - if let Some(leaf) = wait_for_first_leaf(ctx).await? { + if let Some(leaf) = { wait_for_first_leaf(ctx) }.await? { + let sender = ctx.sender().clone(); Ok(Some(( leaf.clone(), - RollingSessionWindow::new(ctx.sender().clone(), DISPUTE_WINDOW, leaf.hash) + RollingSessionWindow::new(sender, DISPUTE_WINDOW, leaf.hash) .await .map_err(JfyiError::RollingSessionWindow)?, ))) diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index dfdcc1698efc..914c5e961f26 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -332,7 +332,7 @@ where } } -#[overseer::subsystem(RuntimeApi, error = SubsystemError, prefix = self::overseer)] +#[overseer::contextbounds(RuntimeApi, prefix = self::overseer)] async fn run( mut ctx: Context, mut subsystem: RuntimeApiSubsystem, diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 8985d48afbac..e51bb3595cf0 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -420,7 +420,7 @@ impl State { ); { - let mut sender = ctx.sender(); + let sender = ctx.sender(); for (peer_id, view) in self.peer_views.iter() { let intersection = view.iter().filter(|h| new_hashes.contains(h)); let view_intersection = View::new(intersection.cloned(), view.finalized_number); @@ -713,7 +713,7 @@ impl State { "Unexpected assignment", ); if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { - modify_reputation(ctx, peer_id, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; } } return @@ -738,7 +738,7 @@ impl State { ?message_subject, "Duplicate assignment", ); - modify_reputation(ctx, peer_id, COST_DUPLICATE_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id, COST_DUPLICATE_MESSAGE).await; } return } @@ -750,13 +750,13 @@ impl State { ?message_subject, "Assignment from a peer is out of view", ); - modify_reputation(ctx, peer_id.clone(), COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id.clone(), COST_UNEXPECTED_MESSAGE).await; }, } // if the assignment is known to be valid, reward the peer if entry.knowledge.contains(&message_subject, message_kind) { - modify_reputation(ctx, peer_id.clone(), BENEFIT_VALID_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE).await; if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known assignment"); peer_knowledge.received.insert(message_subject, message_kind); @@ -792,7 +792,7 @@ impl State { ); match result { AssignmentCheckResult::Accepted => { - modify_reputation(ctx, peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST).await; + modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST).await; entry.knowledge.known_messages.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { peer_knowledge.received.insert(message_subject.clone(), message_kind); @@ -820,7 +820,7 @@ impl State { ?peer_id, "Got an assignment too far in the future", ); - modify_reputation(ctx, peer_id, COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE).await; + modify_reputation(ctx.sender(), peer_id, COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE).await; return }, AssignmentCheckResult::Bad(error) => { @@ -831,7 +831,7 @@ impl State { %error, "Got a bad assignment from peer", ); - modify_reputation(ctx, peer_id, COST_INVALID_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id, COST_INVALID_MESSAGE).await; return }, } @@ -963,7 +963,7 @@ impl State { _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { - modify_reputation(ctx, peer_id, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; } } return @@ -982,7 +982,7 @@ impl State { ?message_subject, "Unknown approval assignment", ); - modify_reputation(ctx, peer_id, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; return } @@ -999,7 +999,7 @@ impl State { "Duplicate approval", ); - modify_reputation(ctx, peer_id, COST_DUPLICATE_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id, COST_DUPLICATE_MESSAGE).await; } return } @@ -1011,14 +1011,14 @@ impl State { ?message_subject, "Approval from a peer is out of view", ); - modify_reputation(ctx, peer_id.clone(), COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id.clone(), COST_UNEXPECTED_MESSAGE).await; }, } // if the approval is known to be valid, reward the peer if entry.knowledge.contains(&message_subject, message_kind) { gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known approval"); - modify_reputation(ctx, peer_id.clone(), BENEFIT_VALID_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE).await; if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { peer_knowledge.received.insert(message_subject.clone(), message_kind); } @@ -1049,7 +1049,7 @@ impl State { ); match result { ApprovalCheckResult::Accepted => { - modify_reputation(ctx, peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST).await; + modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST).await; entry.knowledge.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { @@ -1057,7 +1057,7 @@ impl State { } }, ApprovalCheckResult::Bad(error) => { - modify_reputation(ctx, peer_id, COST_INVALID_MESSAGE).await; + modify_reputation(ctx.sender(), peer_id, COST_INVALID_MESSAGE).await; gum::info!( target: LOG_TARGET, ?peer_id, @@ -1568,8 +1568,7 @@ async fn adjust_required_routing_and_propagate(ctx: &mut Context, peer_id: PeerId, rep: Rep) { +async fn modify_reputation(sender: &mut impl overseer::ApprovalDistributionSenderTrait, peer_id: PeerId, rep: Rep) { gum::trace!( target: LOG_TARGET, reputation = ?rep, @@ -1577,7 +1576,7 @@ async fn modify_reputation(ctx: &mut Context, peer_id: PeerId, rep: Rep "Reputation change for peer", ); - ctx.send_message(NetworkBridgeMessage::ReportPeer(peer_id, rep)).await; + sender.send_message(NetworkBridgeMessage::ReportPeer(peer_id, rep)).await; } #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] diff --git a/node/network/availability-distribution/src/requester/session_cache.rs b/node/network/availability-distribution/src/requester/session_cache.rs index 60a7aaf1b345..1609f150e4e6 100644 --- a/node/network/availability-distribution/src/requester/session_cache.rs +++ b/node/network/availability-distribution/src/requester/session_cache.rs @@ -79,6 +79,7 @@ pub struct BadValidators { pub bad_validators: Vec, } +#[overseer::contextbounds(AvailabilityDistribution, prefix = self::overseer)] impl SessionCache { /// Create a new `SessionCache`. pub fn new() -> Self { diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 0745fa4c221c..15e12c7c85e1 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -310,10 +310,10 @@ impl BitfieldDistribution { } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation(ctx: &mut Context, relay_parent: Hash, peer: PeerId, rep: Rep) { +async fn modify_reputation(sender: &mut impl overseer::BitfieldDistributionSenderTrait, relay_parent: Hash, peer: PeerId, rep: Rep) { gum::trace!(target: LOG_TARGET, ?relay_parent, ?rep, %peer, "reputation change"); - ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await + sender.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await } /// Distribute a given valid and signature checked bitfield message. /// @@ -485,7 +485,7 @@ async fn process_incoming_peer_message( ); // we don't care about this, not part of our view. if !state.view.contains(&relay_parent) { - modify_reputation(ctx, relay_parent, origin, COST_NOT_IN_VIEW).await; + modify_reputation(ctx.sender(), relay_parent, origin, COST_NOT_IN_VIEW).await; return } @@ -494,7 +494,7 @@ async fn process_incoming_peer_message( let job_data: &mut _ = if let Some(ref mut job_data) = job_data { job_data } else { - modify_reputation(ctx, relay_parent, origin, COST_NOT_IN_VIEW).await; + modify_reputation(ctx.sender(), relay_parent, origin, COST_NOT_IN_VIEW).await; return }; @@ -511,7 +511,7 @@ async fn process_incoming_peer_message( let validator_set = &job_data.validator_set; if validator_set.is_empty() { gum::trace!(target: LOG_TARGET, ?relay_parent, ?origin, "Validator set is empty",); - modify_reputation(ctx, relay_parent, origin, COST_MISSING_PEER_SESSION_KEY).await; + modify_reputation(ctx.sender(), relay_parent, origin, COST_MISSING_PEER_SESSION_KEY).await; return } @@ -521,7 +521,7 @@ async fn process_incoming_peer_message( let validator = if let Some(validator) = validator_set.get(validator_index.0 as usize) { validator.clone() } else { - modify_reputation(ctx, relay_parent, origin, COST_VALIDATOR_INDEX_INVALID).await; + modify_reputation(ctx.sender(), relay_parent, origin, COST_VALIDATOR_INDEX_INVALID).await; return }; @@ -534,7 +534,7 @@ async fn process_incoming_peer_message( received_set.insert(validator.clone()); } else { gum::trace!(target: LOG_TARGET, ?validator_index, ?origin, "Duplicate message"); - modify_reputation(ctx, relay_parent, origin, COST_PEER_DUPLICATE_MESSAGE).await; + modify_reputation(ctx.sender(), relay_parent, origin, COST_PEER_DUPLICATE_MESSAGE).await; return }; @@ -548,13 +548,13 @@ async fn process_incoming_peer_message( "already received a message for validator", ); if old_message.signed_availability.as_unchecked() == &bitfield { - modify_reputation(ctx, relay_parent, origin, BENEFIT_VALID_MESSAGE).await; + modify_reputation(ctx.sender(), relay_parent, origin, BENEFIT_VALID_MESSAGE).await; } return } let signed_availability = match bitfield.try_into_checked(&signing_context, &validator) { Err(_) => { - modify_reputation(ctx, relay_parent, origin, COST_SIGNATURE_INVALID).await; + modify_reputation(ctx.sender(), relay_parent, origin, COST_SIGNATURE_INVALID).await; return }, Ok(bitfield) => bitfield, @@ -580,7 +580,7 @@ async fn process_incoming_peer_message( ) .await; - modify_reputation(ctx, relay_parent, origin, BENEFIT_VALID_MESSAGE_FIRST).await + modify_reputation(ctx.sender(), relay_parent, origin, BENEFIT_VALID_MESSAGE_FIRST).await } /// Deal with network bridge updates and track what needs to be tracked diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 3fd59596e41a..3bf59e043414 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -1056,55 +1056,49 @@ async fn dispatch_collation_event_to_all( dispatch_collation_events_to_all(std::iter::once(event), ctx).await } + fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl overseer::NetworkBridgeSenderTrait, + sender: &mut impl overseer::NetworkBridgeSenderTrait, ) { - match event { - NetworkBridgeEvent::::BitfieldDistribution( - bdm, - ) => ctx.send_unbounded_message(bdm), - NetworkBridgeEvent::::StatementDistribution( - sdm, - ) => ctx.send_unbounded_message(sdm), - NetworkBridgeEvent::::ApprovalDistribution( - adm, - ) => ctx.send_unbounded_message(adm), - } + event.focus() + .ok() + .map(ApprovalDistributionMessage::from) + .and_then(|msg| Some(sender.send_unbounded_message(msg))); + + event.focus() + .ok() + .map(BitfieldDistributionMessage::from) + .and_then(|msg| Some(sender.send_unbounded_message(msg))); + + event.focus() + .ok() + .map(StatementDistributionMessage::from) + .and_then(|msg| Some(sender.send_unbounded_message(msg))); } fn dispatch_collation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl overseer::NetworkBridgeSenderTrait, + sender: &mut impl overseer::NetworkBridgeSenderTrait, ) { - match event { - NetworkBridgeEvent::::BitfieldDistribution( - bdm, - ) => ctx.send_unbounded_message(bdm), - NetworkBridgeEvent::::StatementDistribution( - sdm, - ) => ctx.send_unbounded_message(sdm), - NetworkBridgeEvent::::ApprovalDistribution( - adm, - ) => ctx.send_unbounded_message(adm), + if let Ok(msg) = event.focus() { + sender.send_unbounded_message(CollatorProtocolMessage::NetworkBridgeUpdate(msg)) } } async fn dispatch_validation_events_to_all( events: I, - ctx: &mut impl overseer::NetworkBridgeSenderTrait, + sender: &mut impl overseer::NetworkBridgeSenderTrait, ) where I: IntoIterator>, I::IntoIter: Send, { - let sender = ctx.sender(); for event in events { - sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; - sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; - sender - .send_messages(event.focus().map(StatementDistributionMessage::from)) + sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; + sender.send_messages(event.focus().map(StatementDistributionMessage::from)) .await; + sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; } } diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 86e9e5a8e5a9..e5ffbdb9c9d9 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -362,6 +362,7 @@ impl State { /// or the relay-parent isn't in the active-leaves set, we ignore the message /// as it must be invalid in that case - although this indicates a logic error /// elsewhere in the node. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn distribute_collation( ctx: &mut Context, runtime: &mut RuntimeInfo, @@ -371,10 +372,6 @@ async fn distribute_collation( pov: PoV, result_sender: Option>, ) -> Result<()> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { let relay_parent = receipt.descriptor.relay_parent; @@ -492,6 +489,7 @@ struct GroupValidators { /// Figure out current group of validators assigned to the para being collated on. /// /// Returns [`ValidatorId`]'s of current group as determined based on the `relay_parent`. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn determine_our_validators( ctx: &mut Context, runtime: &mut RuntimeInfo, @@ -499,10 +497,6 @@ async fn determine_our_validators( cores: usize, relay_parent: Hash, ) -> Result -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { let session_index = runtime.get_session_index_for_child(ctx.sender(), relay_parent).await?; let info = &runtime @@ -530,11 +524,8 @@ where } /// Issue a `Declare` collation message to the given `peer`. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { let declare_signature_payload = protocol_v1::declare_signature_payload(&state.local_peer_id); @@ -555,11 +546,8 @@ where /// Issue a connection request to a set of validators and /// revoke the previous connection request. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn connect_to_validators(ctx: &mut Context, validator_ids: Vec) -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { // ignore address resolution failure // will reissue a new request on new collation @@ -576,15 +564,13 @@ where /// /// This will only advertise a collation if there exists one for the given `relay_parent` and the given `peer` is /// set as validator for our para at the given `relay_parent`. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn advertise_collation( ctx: &mut Context, state: &mut State, relay_parent: Hash, peer: PeerId, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { let should_advertise = state .our_validators_groups @@ -638,16 +624,13 @@ async fn advertise_collation( } /// The main incoming message dispatching switch. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn process_msg( ctx: &mut Context, runtime: &mut RuntimeInfo, state: &mut State, msg: CollatorProtocolMessage, ) -> Result<()> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { use CollatorProtocolMessage::*; @@ -752,6 +735,7 @@ async fn send_collation( } /// A networking messages switch. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_incoming_peer_message( ctx: &mut Context, runtime: &mut RuntimeInfo, @@ -759,10 +743,6 @@ async fn handle_incoming_peer_message( origin: PeerId, msg: protocol_v1::CollatorProtocolMessage, ) -> Result<()> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { use protocol_v1::CollatorProtocolMessage::*; @@ -836,15 +816,12 @@ where } /// Process an incoming network request for a collation. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_incoming_request( ctx: &mut Context, state: &mut State, req: IncomingRequest, ) -> Result<()> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { let _span = state .span_per_relay_parent @@ -913,15 +890,13 @@ where } /// Our view has changed. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_peer_view_change( ctx: &mut Context, state: &mut State, peer_id: PeerId, view: View, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { let current = state.peer_views.entry(peer_id.clone()).or_default(); @@ -935,16 +910,13 @@ async fn handle_peer_view_change( } /// Bridge messages switch. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_network_msg( ctx: &mut Context, runtime: &mut RuntimeInfo, state: &mut State, bridge_message: NetworkBridgeEvent, ) -> Result<()> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { use NetworkBridgeEvent::*; @@ -1029,6 +1001,7 @@ async fn handle_our_view_change(state: &mut State, view: OurView) -> Result<()> } /// The collator protocol collator side main loop. +#[overseer::contextbounds(CollatorProtocol, prefix = crate::overseer)] pub(crate) async fn run( mut ctx: Context, local_peer_id: PeerId, @@ -1036,10 +1009,6 @@ pub(crate) async fn run( mut req_receiver: IncomingRequestReceiver, metrics: Metrics, ) -> std::result::Result<(), FatalError> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { use OverseerSignal::*; diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index fe57017b6f60..2cd662d90107 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -89,6 +89,7 @@ pub struct CollatorProtocolSubsystem { protocol_side: ProtocolSide, } +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] impl CollatorProtocolSubsystem { /// Start the collator protocol. /// If `id` is `Some` this is a collator side of the protocol. @@ -99,10 +100,6 @@ impl CollatorProtocolSubsystem { } async fn run(self, ctx: Context) -> std::result::Result<(), error::FatalError> - where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => @@ -126,11 +123,7 @@ impl CollatorProtocolSubsystem { } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation(ctx: &mut Context, peer: PeerId, rep: Rep) -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +async fn modify_reputation(sender: &mut impl overseer::CollatorProtocolSenderTrait, peer: PeerId, rep: Rep) { gum::trace!( target: LOG_TARGET, @@ -139,5 +132,5 @@ where "reputation change for peer", ); - ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await; + sender.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await; } diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index d11637b2736b..a79e6c6e14e3 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -630,26 +630,19 @@ fn collator_peer_id( }) } -async fn disconnect_peer(ctx: &mut Context, peer_id: PeerId) -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +async fn disconnect_peer(sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_id: PeerId) { - ctx.send_message(NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation)) + sender.send_message(NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation)) .await } /// Another subsystem has requested to fetch collations on a particular leaf for some para. -async fn fetch_collation( - ctx: &mut Context, +async fn fetch_collation( + sender: &mut impl overseer::CollatorProtocolSenderTrait, state: &mut State, pc: PendingCollation, id: CollatorId, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { let (tx, rx) = oneshot::channel(); @@ -665,7 +658,7 @@ async fn fetch_collation( if let Some(peer_data) = state.peer_data.get(&peer_id) { if peer_data.has_advertised(&relay_parent) { - request_collation(ctx, state, relay_parent, para_id, peer_id, tx).await; + request_collation(sender, state, relay_parent, para_id, peer_id, tx).await; } else { gum::debug!( target: LOG_TARGET, @@ -689,55 +682,46 @@ async fn fetch_collation( } /// Report a collator for some malicious actions. -async fn report_collator( - ctx: &mut Context, +async fn report_collator( + sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_data: &HashMap, id: CollatorId, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { if let Some(peer_id) = collator_peer_id(peer_data, &id) { - modify_reputation(ctx, peer_id, COST_REPORT_BAD).await; + modify_reputation(sender, peer_id, COST_REPORT_BAD).await; } } /// Some other subsystem has reported a collator as a good one, bump reputation. -async fn note_good_collation( - ctx: &mut Context, +async fn note_good_collation( + sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_data: &HashMap, id: CollatorId, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { if let Some(peer_id) = collator_peer_id(peer_data, &id) { - modify_reputation(ctx, peer_id, BENEFIT_NOTIFY_GOOD).await; + modify_reputation(sender, peer_id, BENEFIT_NOTIFY_GOOD).await; } } /// Notify a collator that its collation got seconded. -async fn notify_collation_seconded( - ctx: &mut Context, +async fn notify_collation_seconded( + sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_id: PeerId, relay_parent: Hash, statement: SignedFullStatement, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { let wire_message = protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement.into()); - ctx.send_message(NetworkBridgeMessage::SendCollationMessage( + sender.send_message(NetworkBridgeMessage::SendCollationMessage( vec![peer_id], Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)), )) .await; - modify_reputation(ctx, peer_id, BENEFIT_NOTIFY_GOOD).await; + modify_reputation(sender, peer_id, BENEFIT_NOTIFY_GOOD).await; } /// A peer's view has changed. A number of things should be done: @@ -760,17 +744,14 @@ async fn handle_peer_view_change(state: &mut State, peer_id: PeerId, view: View) /// - Check if the requested collation is in our view. /// - Update `PerRequest` records with the `result` field if necessary. /// And as such invocations of this function may rely on that. -async fn request_collation( - ctx: &mut Context, +async fn request_collation( + sender: &mut impl overseer::CollatorProtocolSenderTrait, state: &mut State, relay_parent: Hash, para_id: ParaId, peer_id: PeerId, result: oneshot::Sender<(CandidateReceipt, PoV)>, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { if !state.view.contains(&relay_parent) { gum::debug!( @@ -822,7 +803,7 @@ async fn request_collation( "Requesting collation", ); - ctx.send_message(NetworkBridgeMessage::SendRequests( + sender.send_message(NetworkBridgeMessage::SendRequests( vec![requests], IfDisconnected::ImmediateError, )) @@ -830,22 +811,20 @@ async fn request_collation( } /// Networking message has been received. +#[overseer::contextbounds(CollatorProtocol, prefix = overseer)] async fn process_incoming_peer_message( ctx: &mut Context, state: &mut State, origin: PeerId, msg: protocol_v1::CollatorProtocolMessage, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { use protocol_v1::CollatorProtocolMessage::*; use sp_runtime::traits::AppVerify; match msg { Declare(collator_id, para_id, signature) => { if collator_peer_id(&state.peer_data, &collator_id).is_some() { - modify_reputation(ctx, origin, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; return } @@ -858,7 +837,7 @@ async fn process_incoming_peer_message( ?para_id, "Unknown peer", ); - modify_reputation(ctx, origin, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; return }, }; @@ -870,7 +849,7 @@ async fn process_incoming_peer_message( ?para_id, "Peer is not in the collating state", ); - modify_reputation(ctx, origin, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; return } @@ -881,7 +860,7 @@ async fn process_incoming_peer_message( ?para_id, "Signature verification failure", ); - modify_reputation(ctx, origin, COST_INVALID_SIGNATURE).await; + modify_reputation(ctx.sender(), origin, COST_INVALID_SIGNATURE).await; return } @@ -904,9 +883,9 @@ async fn process_incoming_peer_message( "Declared as collator for unneeded para", ); - modify_reputation(ctx, origin.clone(), COST_UNNEEDED_COLLATOR).await; + modify_reputation(ctx.sender(), origin.clone(), COST_UNNEEDED_COLLATOR).await; gum::trace!(target: LOG_TARGET, "Disconnecting unneeded collator"); - disconnect_peer(ctx, origin).await; + disconnect_peer(ctx.sender(), origin).await; } }, AdvertiseCollation(relay_parent) => { @@ -922,7 +901,7 @@ async fn process_incoming_peer_message( "Advertise collation out of view", ); - modify_reputation(ctx, origin, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; return } @@ -934,7 +913,7 @@ async fn process_incoming_peer_message( ?relay_parent, "Advertise collation message has been received from an unknown peer", ); - modify_reputation(ctx, origin, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; return }, Some(p) => p, @@ -970,7 +949,7 @@ async fn process_incoming_peer_message( collations.status = CollationStatus::Fetching; collations.waiting_collation = Some(id.clone()); - fetch_collation(ctx, state, pending_collation.clone(), id).await; + fetch_collation(ctx.sender(), state, pending_collation.clone(), id).await; }, CollationStatus::Seconded => { gum::trace!( @@ -992,7 +971,7 @@ async fn process_incoming_peer_message( "Invalid advertisement", ); - modify_reputation(ctx, origin, COST_UNEXPECTED_MESSAGE).await; + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; }, } }, @@ -1019,16 +998,13 @@ async fn remove_relay_parent(state: &mut State, relay_parent: Hash) -> Result<() } /// Our view has changed. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_our_view_change( ctx: &mut Context, state: &mut State, keystore: &SyncCryptoStorePtr, view: OurView, ) -> Result<()> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { let old_view = std::mem::replace(&mut state.view, view); @@ -1070,7 +1046,7 @@ where ?para_id, "Disconnecting peer on view change (not current parachain id)" ); - disconnect_peer(ctx, peer_id.clone()).await; + disconnect_peer(ctx.sender(), peer_id.clone()).await; } } } @@ -1079,16 +1055,13 @@ where } /// Bridge event switch. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_network_msg( ctx: &mut Context, state: &mut State, keystore: &SyncCryptoStorePtr, bridge_message: NetworkBridgeEvent, ) -> Result<()> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { use NetworkBridgeEvent::*; @@ -1119,15 +1092,13 @@ where } /// The main message receiver switch. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn process_msg( ctx: &mut Context, keystore: &SyncCryptoStorePtr, msg: CollatorProtocolMessage, state: &mut State, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { use CollatorProtocolMessage::*; @@ -1148,7 +1119,7 @@ async fn process_msg( ); }, ReportCollator(id) => { - report_collator(ctx, &state.peer_data, id).await; + report_collator(ctx.sender(), &state.peer_data, id).await; }, NetworkBridgeUpdate(event) => { if let Err(e) = handle_network_msg(ctx, state, keystore, event).await { @@ -1163,8 +1134,8 @@ async fn process_msg( if let Some(collation_event) = state.pending_candidates.remove(&parent) { let (collator_id, pending_collation) = collation_event; let PendingCollation { relay_parent, peer_id, .. } = pending_collation; - note_good_collation(ctx, &state.peer_data, collator_id).await; - notify_collation_seconded(ctx, peer_id, relay_parent, stmt).await; + note_good_collation(ctx.sender(), &state.peer_data, collator_id).await; + notify_collation_seconded(ctx.sender(), peer_id, relay_parent, stmt).await; if let Some(collations) = state.collations_per_relay_parent.get_mut(&parent) { collations.status = CollationStatus::Seconded; @@ -1195,7 +1166,7 @@ async fn process_msg( Entry::Vacant(_) => return, }; - report_collator(ctx, &state.peer_data, id.clone()).await; + report_collator(ctx.sender(), &state.peer_data, id.clone()).await; dequeue_next_collation_and_fetch(ctx, state, parent, id).await; }, @@ -1222,16 +1193,13 @@ fn infinite_stream(every: Duration) -> impl FusedStream { } /// The main run loop. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] pub(crate) async fn run( mut ctx: Context, keystore: SyncCryptoStorePtr, eviction_policy: crate::CollatorEvictionPolicy, metrics: Metrics, ) -> std::result::Result<(), crate::error::FatalError> -where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, { let mut state = State { metrics, ..Default::default() }; @@ -1259,7 +1227,7 @@ where } } _ = next_inactivity_stream.next() => { - disconnect_inactive_peers(&mut ctx, &eviction_policy, &state.peer_data).await; + disconnect_inactive_peers(ctx.sender(), &eviction_policy, &state.peer_data).await; } res = state.collation_fetches.select_next_some() => { handle_collation_fetched_result(&mut ctx, &mut state, res).await; @@ -1282,7 +1250,7 @@ where ).await; for (peer_id, rep) in reputation_changes { - modify_reputation(&mut ctx, peer_id, rep).await; + modify_reputation(ctx.sender(), peer_id, rep).await; } }, } @@ -1316,16 +1284,14 @@ async fn poll_requests( } /// Dequeue another collation and fetch. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn dequeue_next_collation_and_fetch( ctx: &mut Context, state: &mut State, relay_parent: Hash, // The collator we tried to fetch from last. previous_fetch: CollatorId, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { if let Some((next, id)) = state .collations_per_relay_parent @@ -1338,7 +1304,7 @@ async fn dequeue_next_collation_and_fetch( ?id, "Successfully dequeued next advertisement - fetching ..." ); - fetch_collation(ctx, state, next, id).await; + fetch_collation(ctx.sender(), state, next, id).await; } else { gum::debug!( target: LOG_TARGET, @@ -1350,14 +1316,12 @@ async fn dequeue_next_collation_and_fetch( } /// Handle a fetched collation result. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_collation_fetched_result( ctx: &mut Context, state: &mut State, (mut collation_event, res): PendingCollationFetch, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { // If no prior collation for this relay parent has been seconded, then // memorize the `collation_event` for that `relay_parent`, such that we may @@ -1396,7 +1360,7 @@ async fn handle_collation_fetched_result( if let Entry::Vacant(entry) = state.pending_candidates.entry(relay_parent) { collation_event.1.commitments_hash = Some(candidate_receipt.commitments_hash); - ctx.send_message(CandidateBackingMessage::Second( + ctx.sender().send_message(CandidateBackingMessage::Second( relay_parent.clone(), candidate_receipt, pov, @@ -1417,19 +1381,16 @@ async fn handle_collation_fetched_result( // This issues `NetworkBridge` notifications to disconnect from all inactive peers at the // earliest possible point. This does not yet clean up any metadata, as that will be done upon // receipt of the `PeerDisconnected` event. -async fn disconnect_inactive_peers( - ctx: &mut Context, +async fn disconnect_inactive_peers( + sender: &mut impl overseer::CollatorProtocolSenderTrait, eviction_policy: &crate::CollatorEvictionPolicy, peers: &HashMap, -) where - Context: overseer::CollatorProtocolContextTrait, - ::Sender: - overseer::CollatorProtocolSenderTrait, +) { for (peer, peer_data) in peers { if peer_data.is_inactive(&eviction_policy) { gum::trace!(target: LOG_TARGET, "Disconnecting inactive peer"); - disconnect_peer(ctx, peer.clone()).await; + disconnect_peer(sender, peer.clone()).await; } } } diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 0ee5eaac5e7c..fbdeafa86cf4 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -1158,8 +1158,8 @@ async fn send_statements( } } -async fn report_peer(ctx: &mut Context, peer: PeerId, rep: Rep) { - ctx.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await +async fn report_peer(sender: &mut impl overseer::StatementDistributionSenderTrait, peer: PeerId, rep: Rep) { + sender.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await } /// If message contains a statement, then retrieve it, otherwise fork task to fetch it. @@ -1377,7 +1377,7 @@ async fn handle_incoming_message<'a, Context>( ); if !recent_outdated_heads.is_recent_outdated(&relay_parent) { - report_peer(ctx, peer, COST_UNEXPECTED_STATEMENT).await; + report_peer(ctx.sender(), peer, COST_UNEXPECTED_STATEMENT).await; } return None @@ -1387,7 +1387,7 @@ async fn handle_incoming_message<'a, Context>( if let protocol_v1::StatementDistributionMessage::LargeStatement(_) = message { if let Err(rep) = peer_data.receive_large_statement(&relay_parent) { gum::debug!(target: LOG_TARGET, ?peer, ?message, ?rep, "Unexpected large statement.",); - report_peer(ctx, peer, rep).await; + report_peer(ctx.sender(), peer, rep).await; return None } } @@ -1428,16 +1428,16 @@ async fn handle_incoming_message<'a, Context>( // Report peer merely if this is not a duplicate out-of-view statement that // was caused by a missing Seconded statement from this peer if unexpected_count == 0_usize { - report_peer(ctx, peer, rep).await; + report_peer(ctx.sender(), peer, rep).await; } }, // This happens when we have an unexpected remote peer that announced Seconded COST_UNEXPECTED_STATEMENT_REMOTE => { metrics.on_unexpected_statement_seconded(); - report_peer(ctx, peer, rep).await; + report_peer(ctx.sender(), peer, rep).await; }, _ => { - report_peer(ctx, peer, rep).await; + report_peer(ctx.sender(), peer, rep).await; }, } @@ -1458,7 +1458,7 @@ async fn handle_incoming_message<'a, Context>( peer_data .receive(&relay_parent, &fingerprint, max_message_count) .expect("checked in `check_can_receive` above; qed"); - report_peer(ctx, peer, BENEFIT_VALID_STATEMENT).await; + report_peer(ctx.sender(), peer, BENEFIT_VALID_STATEMENT).await; return None }, @@ -1468,7 +1468,7 @@ async fn handle_incoming_message<'a, Context>( match check_statement_signature(&active_head, relay_parent, unchecked_compact) { Err(statement) => { gum::debug!(target: LOG_TARGET, ?peer, ?statement, "Invalid statement signature"); - report_peer(ctx, peer, COST_INVALID_SIGNATURE).await; + report_peer(ctx.sender(), peer, COST_INVALID_SIGNATURE).await; return None }, Ok(statement) => statement, @@ -1494,7 +1494,7 @@ async fn handle_incoming_message<'a, Context>( is_large_statement, "Full statement had bad payload." ); - report_peer(ctx, peer, COST_WRONG_HASH).await; + report_peer(ctx.sender(), peer, COST_WRONG_HASH).await; return None }, Ok(statement) => statement, @@ -1533,7 +1533,7 @@ async fn handle_incoming_message<'a, Context>( unreachable!("checked in `is_useful_or_unknown` above; qed"); }, NotedStatement::Fresh(statement) => { - report_peer(ctx, peer, BENEFIT_VALID_STATEMENT_FIRST).await; + report_peer(ctx.sender(), peer, BENEFIT_VALID_STATEMENT_FIRST).await; let mut _span = handle_incoming_span.child("notify-backing"); @@ -1848,9 +1848,9 @@ impl StatementDistributionSubsystem { bad_peers, } => { for bad in bad_peers { - report_peer(ctx, bad, COST_FETCH_FAIL).await; + report_peer(ctx.sender(), bad, COST_FETCH_FAIL).await; } - report_peer(ctx, from_peer, BENEFIT_VALID_RESPONSE).await; + report_peer(ctx.sender(), from_peer, BENEFIT_VALID_RESPONSE).await; let active_head = active_heads .get_mut(&relay_parent) @@ -1932,7 +1932,7 @@ impl StatementDistributionSubsystem { } } }, - RequesterMessage::ReportPeer(peer, rep) => report_peer(ctx, peer, rep).await, + RequesterMessage::ReportPeer(peer, rep) => report_peer(ctx.sender(), peer, rep).await, } Ok(()) } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 3b7f90e0f383..bcf486d2a0db 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -494,9 +494,9 @@ pub struct Overseer { availability_store: AvailabilityStore, #[subsystem(NetworkBridgeMessage, sends: [ - ApprovalDistributionMessage, - StatementDistributionMessage, BitfieldDistributionMessage, + StatementDistributionMessage, + ApprovalDistributionMessage, GossipSupportMessage, DisputeDistributionMessage, CollationGenerationMessage, diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index af50c481c6ff..632dc429f7fe 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -83,6 +83,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-rpc = { path = "../../rpc" } polkadot-node-subsystem = {path = "../subsystem" } polkadot-node-subsystem-util = { path = "../subsystem-util" } +polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-runtime-parachains = { path = "../../runtime/parachains" } polkadot-node-network-protocol = { path = "../network/protocol" } diff --git a/node/service/src/overseer.rs b/node/service/src/overseer.rs index aa90abb91fb2..392d042dd4b9 100644 --- a/node/service/src/overseer.rs +++ b/node/service/src/overseer.rs @@ -31,8 +31,10 @@ pub use polkadot_overseer::{ }; use polkadot_overseer::{ metrics::Metrics as OverseerMetrics, BlockInfo, InitializedOverseerBuilder, MetricsTrait, - Overseer, OverseerConnector, OverseerHandle, + Overseer, OverseerConnector, OverseerHandle, OverseerSubsystemContext, }; +use polkadot_node_subsystem_types::messages::{BitfieldSigningMessage, ProvisionerMessage}; +use polkadot_overseer::gen::SubsystemContext; use polkadot_primitives::runtime_api::ParachainHost; use sc_authority_discovery::Service as AuthorityDiscoveryService; @@ -152,9 +154,9 @@ pub fn prepared_overseer_builder<'a, Spawner, RuntimeClient>( StatementDistributionSubsystem, AvailabilityDistributionSubsystem, AvailabilityRecoverySubsystem, - BitfieldSigningSubsystem, + BitfieldSigningSubsystem as SubsystemContext>::Sender>, BitfieldDistributionSubsystem, - ProvisionerSubsystem, + ProvisionerSubsystem as SubsystemContext>::Sender>, RuntimeApiSubsystem, AvailabilityStoreSubsystem, NetworkBridgeSubsystem< From 031ad5b16795a604699b90fe6bba7c00873b1fe3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 5 May 2022 16:52:24 +0200 Subject: [PATCH 096/139] foo --- node/malus/src/interceptor.rs | 39 +++++++++++++------ node/malus/src/tests.rs | 14 +++---- node/malus/src/variants/common.rs | 8 ++-- .../src/variants/suggest_garbage_candidate.rs | 9 ++--- 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/node/malus/src/interceptor.rs b/node/malus/src/interceptor.rs index ae68ea9807c5..5a7f3b3aa1c1 100644 --- a/node/malus/src/interceptor.rs +++ b/node/malus/src/interceptor.rs @@ -21,16 +21,21 @@ //! messages on the overseer level. use polkadot_node_subsystem::*; -pub use polkadot_node_subsystem::{overseer, FromOverseer}; +pub use polkadot_node_subsystem::{messages, overseer, FromOverseer}; use std::{future::Future, pin::Pin}; /// Filter incoming and outgoing messages. pub trait MessageInterceptor: Send + Sync + Clone + 'static where - Sender: overseer::SubsystemSender + Clone + 'static, + Sender: + overseer::SubsystemSender + + overseer::SubsystemSender< + ::OutgoingMessages + > + + Clone + 'static, { /// The message type the original subsystem handles incoming. - type Message: AssociateOutgoing + Send + 'static; + type Message: overseer::AssociateOutgoing + Send + 'static; /// Filter messages that are to be received by /// the subsystem. @@ -46,10 +51,7 @@ where } /// Modify outgoing messages. - fn intercept_outgoing( - &self, - msg: AssociateOutgoing::OutgoingMessages, - ) -> Option { + fn intercept_outgoing(&self, msg: ::OutgoingMessages) -> Option<::OutgoingMessages> { Some(msg) } } @@ -64,7 +66,8 @@ pub struct InterceptedSender { #[async_trait::async_trait] impl overseer::SubsystemSender for InterceptedSender where - Sender: overseer::SubsystemSender<>::Message>, + ::OutgoingMessages: From, + Sender: overseer::SubsystemSender + overseer::SubsystemSender<::OutgoingMessages>, Fil: MessageInterceptor, { async fn send_message(&mut self, msg: M) { @@ -84,7 +87,7 @@ where } fn send_unbounded_message(&mut self, msg: M) { - if let Some(msg) = self.message_filter.intercept_outgoing(msg) { + if let Some(msg) = self.message_filter.intercept_outgoing(<::OutgoingMessages as From>::from(msg.into)) { self.inner.send_unbounded_message(msg); } } @@ -98,6 +101,11 @@ where ::Sender: overseer::SubsystemSender< ::Sender>>::Message, >, + ::Sender: overseer::SubsystemSender< + < + ::Sender> + >::Message as AssociateOutgoing + >::OutgoingMessages>, { inner: Context, message_filter: Fil, @@ -111,9 +119,13 @@ where ::Sender, Message = ::Message, >, + ::Message: AssociateOutgoing, ::Sender: overseer::SubsystemSender< ::Sender>>::Message, >, + ::Sender: overseer::SubsystemSender< + <::Message as overseer::AssociateOutgoing>::OutgoingMessages + > { pub fn new(mut inner: Context, message_filter: Fil) -> Self { let sender = InterceptedSender::<::Sender, Fil> { @@ -132,13 +144,15 @@ where ::Sender, Message = ::Message, >, - ::Sender: overseer::SubsystemSender< - ::Sender>>::Message, - >, + ::Sender: + overseer::SubsystemSender< + ::Sender>>::Message, + >, { type Message = ::Message; type Sender = InterceptedSender<::Sender, Fil>; type Error = ::Error; + type OutgoingMessages = <::Message as overseer::AssociateOutgoing>::OutgoingMessages; type Signal = ::Signal; async fn try_recv(&mut self) -> Result>, ()> { @@ -209,6 +223,7 @@ where ::Sender: overseer::SubsystemSender< ::Sender>>::Message, >, + ::Message: AssociateOutgoing, { fn start(self, ctx: Context) -> SpawnedSubsystem { let ctx = InterceptedContext::new(ctx, self.message_interceptor); diff --git a/node/malus/src/tests.rs b/node/malus/src/tests.rs index 57072336852b..eb89acc0667d 100644 --- a/node/malus/src/tests.rs +++ b/node/malus/src/tests.rs @@ -19,8 +19,8 @@ use super::*; use polkadot_node_subsystem_test_helpers::*; use polkadot_node_subsystem::{ - messages::{AllMessages, AvailabilityStoreMessage}, - overseer::{dummy::DummySubsystem, gen::TimeoutExt, Subsystem}, + messages::AvailabilityStoreMessage, + overseer::{dummy::DummySubsystem, gen::TimeoutExt, Subsystem, AssociateOutgoing}, SubsystemError, }; @@ -29,8 +29,7 @@ struct BlackHoleInterceptor; impl MessageInterceptor for BlackHoleInterceptor where - Sender: overseer::SubsystemSender - + overseer::SubsystemSender + Sender: overseer::AvailabilityStoreSenderTrait + Clone + 'static, { @@ -53,8 +52,7 @@ struct PassInterceptor; impl MessageInterceptor for PassInterceptor where - Sender: overseer::SubsystemSender - + overseer::SubsystemSender + Sender: overseer::AvailabilityStoreSenderTrait + Clone + 'static, { @@ -68,8 +66,8 @@ async fn overseer_send>(overseer: &mut TestSubsystemContext fn launch_harness(test_gen: G) where F: Future> + Send, - M: Into + std::fmt::Debug + Send + 'static, - AllMessages: From, + M: AssociateOutgoing + Into<::OutgoingMessages> + std::fmt::Debug + Send + 'static, + ::OutgoingMessages: From, Sub: Subsystem, SubsystemError>, G: Fn(TestSubsystemContextHandle) -> (F, Sub), { diff --git a/node/malus/src/variants/common.rs b/node/malus/src/variants/common.rs index 8bed137126d1..b1f951ab948f 100644 --- a/node/malus/src/variants/common.rs +++ b/node/malus/src/variants/common.rs @@ -131,8 +131,7 @@ where subsystem_sender: Sender, response_sender: oneshot::Sender>, ) where - Sender: overseer::SubsystemSender - + overseer::SubsystemSender + Sender: overseer::CandidateValidationSenderTrait + Clone + Send + 'static, @@ -200,8 +199,7 @@ fn create_validation_response( impl MessageInterceptor for ReplaceValidationResult where - Sender: overseer::SubsystemSender - + overseer::SubsystemSender + Sender: overseer::CandidateValidationSenderTrait + Clone + Send + 'static, @@ -336,7 +334,7 @@ where } } - fn intercept_outgoing(&self, msg: AllMessages) -> Option { + fn intercept_outgoing(&self, msg: ::OutgoingMessages) -> Option<::OutgoingMessages> { Some(msg) } } diff --git a/node/malus/src/variants/suggest_garbage_candidate.rs b/node/malus/src/variants/suggest_garbage_candidate.rs index af8191fe10a5..dc51c765140a 100644 --- a/node/malus/src/variants/suggest_garbage_candidate.rs +++ b/node/malus/src/variants/suggest_garbage_candidate.rs @@ -71,8 +71,7 @@ struct NoteCandidate { impl MessageInterceptor for NoteCandidate where - Sender: overseer::SubsystemSender - + overseer::SubsystemSender + Sender: overseer::CandidateBackingSenderTrait + Clone + Send + 'static, @@ -219,9 +218,9 @@ where } } - fn intercept_outgoing(&self, msg: AllMessages) -> Option { + fn intercept_outgoing(&self, msg: overseer::CandidateBackingOutgoingMessages) -> Option { let msg = match msg { - AllMessages::CollatorProtocol(CollatorProtocolMessage::Seconded( + overseer::CandidateBackingOutgoingMessages::CollatorProtocol(CollatorProtocolMessage::Seconded( relay_parent, statement, )) => { @@ -229,7 +228,7 @@ where // TODO: Fix this error. We get this on colaltors because `malicious backing` creates a candidate that gets backed/included. // It is harmless for test parachain collators, but it will prevent cumulus based collators to make progress // as they wait for the relay chain to confirm the seconding of the collation. - AllMessages::CollatorProtocol(CollatorProtocolMessage::Seconded( + overseer::CandidateBackingOutgoingMessages::CollatorProtocol(CollatorProtocolMessage::Seconded( relay_parent, statement, )) From 17de9924378385b0060b62403c0bab41e7103b71 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 5 May 2022 18:22:49 +0200 Subject: [PATCH 097/139] interceptor WIP fixup --- node/malus/src/interceptor.rs | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/node/malus/src/interceptor.rs b/node/malus/src/interceptor.rs index 5a7f3b3aa1c1..80560aebfd03 100644 --- a/node/malus/src/interceptor.rs +++ b/node/malus/src/interceptor.rs @@ -28,8 +28,7 @@ use std::{future::Future, pin::Pin}; pub trait MessageInterceptor: Send + Sync + Clone + 'static where Sender: - overseer::SubsystemSender - + overseer::SubsystemSender< + overseer::SubsystemSender< ::OutgoingMessages > + Clone + 'static, @@ -66,12 +65,15 @@ pub struct InterceptedSender { #[async_trait::async_trait] impl overseer::SubsystemSender for InterceptedSender where + M: overseer::AssociateOutgoing, ::OutgoingMessages: From, - Sender: overseer::SubsystemSender + overseer::SubsystemSender<::OutgoingMessages>, + Sender: overseer::SubsystemSender + + overseer::SubsystemSender<::OutgoingMessages> + + overseer::SubsystemSender<<>::Message as overseer::AssociateOutgoing>::OutgoingMessages>, Fil: MessageInterceptor, { async fn send_message(&mut self, msg: M) { - if let Some(msg) = self.message_filter.intercept_outgoing(msg) { + if let Some(msg) = self.message_filter.intercept_outgoing(<::OutgoingMessages>::from(msg)) { self.inner.send_message(msg).await; } } @@ -82,12 +84,12 @@ where T::IntoIter: Send, { for msg in msgs { - self.send_message(msg).await; + self.send_message(<::OutgoingMessages>::from(msg)).await; } } fn send_unbounded_message(&mut self, msg: M) { - if let Some(msg) = self.message_filter.intercept_outgoing(<::OutgoingMessages as From>::from(msg.into)) { + if let Some(msg) = self.message_filter.intercept_outgoing(<::OutgoingMessages>::from(msg)) { self.inner.send_unbounded_message(msg); } } @@ -96,7 +98,7 @@ where /// A subsystem context, that filters the outgoing messages. pub struct InterceptedContext where - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor<::Sender>, ::Sender: overseer::SubsystemSender< ::Sender>>::Message, @@ -114,7 +116,7 @@ where impl InterceptedContext where - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor< ::Sender, Message = ::Message, @@ -139,19 +141,26 @@ where #[async_trait::async_trait] impl overseer::SubsystemContext for InterceptedContext where - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor< ::Sender, Message = ::Message, >, + ::Sender: + overseer::SubsystemSender< + <::Message as overseer::AssociateOutgoing>::OutgoingMessages + >, ::Sender: overseer::SubsystemSender< ::Sender>>::Message, >, + ::Message: overseer::AssociateOutgoing, + ::Sender: + overseer::SubsystemSender<<::Message as overseer::AssociateOutgoing>::OutgoingMessages>, { type Message = ::Message; type Sender = InterceptedSender<::Sender, Fil>; - type Error = ::Error; + type Error = SubsystemError; type OutgoingMessages = <::Message as overseer::AssociateOutgoing>::OutgoingMessages; type Signal = ::Signal; @@ -213,7 +222,7 @@ impl InterceptedSubsystem { impl overseer::Subsystem for InterceptedSubsystem where - Context: overseer::SubsystemContext + Sync + Send, + Context: overseer::SubsystemContext + Sync + Send, Sub: overseer::Subsystem, SubsystemError>, InterceptedContext: overseer::SubsystemContext, Interceptor: MessageInterceptor< @@ -224,6 +233,8 @@ where ::Sender>>::Message, >, ::Message: AssociateOutgoing, + ::Sender: + overseer::SubsystemSender<<::Message as overseer::AssociateOutgoing>::OutgoingMessages>, { fn start(self, ctx: Context) -> SpawnedSubsystem { let ctx = InterceptedContext::new(ctx, self.message_interceptor); From d47734d86458cc27b8e2b4535428cabac48d4e19 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 6 May 2022 15:54:17 +0200 Subject: [PATCH 098/139] fix warnings --- node/collation-generation/src/lib.rs | 11 +-- node/core/approval-voting/src/import.rs | 4 +- node/core/approval-voting/src/lib.rs | 15 +--- node/core/av-store/src/lib.rs | 3 +- node/core/candidate-validation/src/lib.rs | 4 +- node/core/chain-api/src/lib.rs | 2 +- node/core/chain-selection/src/lib.rs | 2 +- .../dispute-coordinator/src/initialized.rs | 2 +- node/core/dispute-coordinator/src/lib.rs | 3 +- .../src/participation/mod.rs | 2 +- .../src/participation/queues/mod.rs | 2 +- node/core/provisioner/src/lib.rs | 2 +- node/core/pvf-checker/src/lib.rs | 4 +- node/core/runtime-api/src/lib.rs | 3 +- node/malus/src/interceptor.rs | 13 +-- node/malus/src/variants/common.rs | 15 ++-- .../src/variants/suggest_garbage_candidate.rs | 24 +++--- node/network/approval-distribution/src/lib.rs | 18 ++-- .../availability-distribution/src/lib.rs | 2 +- .../src/pov_requester/mod.rs | 8 +- .../src/requester/fetch_task/mod.rs | 10 +-- .../src/requester/mod.rs | 2 +- .../src/requester/session_cache.rs | 2 +- node/network/availability-recovery/src/lib.rs | 10 +-- node/network/bitfield-distribution/src/lib.rs | 9 +- node/network/bridge/src/lib.rs | 18 ++-- .../src/collator_side/mod.rs | 38 ++++----- node/network/collator-protocol/src/lib.rs | 12 +-- .../src/validator_side/mod.rs | 85 +++++++++---------- node/network/dispute-distribution/src/lib.rs | 2 +- .../dispute-distribution/src/receiver/mod.rs | 2 +- .../dispute-distribution/src/sender/mod.rs | 4 +- .../src/sender/send_task.rs | 2 +- node/network/gossip-support/src/lib.rs | 3 +- .../network/statement-distribution/src/lib.rs | 8 +- node/service/src/overseer.rs | 18 ++-- 36 files changed, 174 insertions(+), 190 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index be901037d33e..b68a46335cb3 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -24,7 +24,7 @@ use polkadot_node_primitives::{AvailableData, CollationGenerationConfig, PoV}; use polkadot_node_subsystem::{ messages::{CollationGenerationMessage, CollatorProtocolMessage}, overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, - SubsystemError, SubsystemResult, SubsystemSender, + SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::{ request_availability_cores, request_persisted_validation_data, request_validation_code, @@ -72,8 +72,7 @@ impl CollationGenerationSubsystem { /// /// If `err_tx` is not `None`, errors are forwarded onto that channel as they occur. /// Otherwise, most are logged and then discarded. - async fn run(mut self, mut ctx: Context) - { + async fn run(mut self, mut ctx: Context) { // when we activate new leaves, we spawn a bunch of sub-tasks, each of which is // expected to generate precisely one message. We don't want to block the main loop // at any point waiting for them all, so instead, we create a channel on which they can @@ -107,8 +106,7 @@ impl CollationGenerationSubsystem { incoming: SubsystemResult::Message>>, ctx: &mut Context, sender: &mpsc::Sender, - ) -> bool - { + ) -> bool { match incoming { Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { activated, @@ -177,8 +175,7 @@ async fn handle_new_activations( ctx: &mut Context, metrics: Metrics, sender: &mpsc::Sender, -) -> crate::error::Result<()> -{ +) -> crate::error::Result<()> { // follow the procedure from the guide: // https://w3f.github.io/parachain-implementers-guide/node/collators/collation-generation.html diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index add41c0e4a12..0550ce5ed223 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -38,7 +38,7 @@ use polkadot_node_subsystem::{ ApprovalDistributionMessage, ChainApiMessage, ChainSelectionMessage, RuntimeApiMessage, RuntimeApiRequest, }, - overseer, RuntimeApiError, SubsystemContext, SubsystemError, SubsystemResult, + overseer, RuntimeApiError, SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::{ determine_new_blocks, @@ -46,7 +46,7 @@ use polkadot_node_subsystem_util::{ }; use polkadot_primitives::v2::{ BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, ConsensusLog, CoreIndex, - GroupIndex, Hash, Header, Id as ParaId, SessionIndex, + GroupIndex, Hash, Header, SessionIndex, }; use sc_keystore::LocalKeystore; use sp_consensus_slots::Slot; diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 7da70a82ecda..c53c04c690a5 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -37,9 +37,8 @@ use polkadot_node_subsystem::{ ChainSelectionMessage, DisputeCoordinatorMessage, HighestApprovedAncestorBlock, RuntimeApiMessage, RuntimeApiRequest, }, - overseer::{self, SubsystemSender as _}, - FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, - SubsystemResult, SubsystemSender, + overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, + SubsystemSender, }; use polkadot_node_subsystem_util::{ database::Database, @@ -613,14 +612,8 @@ impl State { match session_window { None => { let sender = ctx.sender().clone(); - self.session_window = Some( - RollingSessionWindow::new( - sender, - APPROVAL_SESSIONS, - head, - ) - .await?, - ); + self.session_window = + Some(RollingSessionWindow::new(sender, APPROVAL_SESSIONS, head).await?); Ok(None) }, Some(mut session_window) => { diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 1ae1cf96691c..086ce0255cf9 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -36,8 +36,7 @@ use polkadot_node_primitives::{AvailableData, ErasureChunk}; use polkadot_node_subsystem::{ errors::{ChainApiError, RuntimeApiError}, messages::{AvailabilityStoreMessage, ChainApiMessage}, - overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, - SubsystemError, + overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_util as util; use polkadot_primitives::v2::{ diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 561977871695..2fdcc976219b 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -35,8 +35,8 @@ use polkadot_node_subsystem::{ CandidateValidationMessage, PreCheckOutcome, RuntimeApiMessage, RuntimeApiRequest, ValidationFailed, }, - overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, - SubsystemResult, SubsystemSender, + overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, + SubsystemSender, }; use polkadot_parachain::primitives::{ValidationParams, ValidationResult as WasmValidationResult}; use polkadot_primitives::v2::{ diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 96bdaa1e0b5a..3543183654f0 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -39,7 +39,7 @@ use sp_blockchain::HeaderBackend; use polkadot_node_subsystem::{ messages::ChainApiMessage, overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, - SubsystemContext, SubsystemError, SubsystemResult, + SubsystemError, SubsystemResult, }; use polkadot_primitives::v2::{Block, BlockId}; diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index 295437a630da..464edf69f13b 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -21,7 +21,7 @@ use polkadot_node_subsystem::{ errors::ChainApiError, messages::{ChainApiMessage, ChainSelectionMessage}, overseer::{self, SubsystemSender}, - FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, + FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::v2::{BlockNumber, ConsensusLog, Hash, Header}; diff --git a/node/core/dispute-coordinator/src/initialized.rs b/node/core/dispute-coordinator/src/initialized.rs index c76bcaed10f3..00817a136334 100644 --- a/node/core/dispute-coordinator/src/initialized.rs +++ b/node/core/dispute-coordinator/src/initialized.rs @@ -34,7 +34,7 @@ use polkadot_node_subsystem::{ BlockDescription, DisputeCoordinatorMessage, DisputeDistributionMessage, ImportStatementsResult, }, - overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SubsystemContext, + overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, OverseerSignal, }; use polkadot_node_subsystem_util::rolling_session_window::{ RollingSessionWindow, SessionWindowUpdate, SessionsUnavailable, diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index f771f0b11e6c..1a7f1540b8b2 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -32,8 +32,7 @@ use sc_keystore::LocalKeystore; use polkadot_node_primitives::{CandidateVotes, DISPUTE_WINDOW}; use polkadot_node_subsystem::{ - overseer, ActivatedLeaf, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, - SubsystemError, + overseer, ActivatedLeaf, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_util::{ database::Database, rolling_session_window::RollingSessionWindow, diff --git a/node/core/dispute-coordinator/src/participation/mod.rs b/node/core/dispute-coordinator/src/participation/mod.rs index fd9bb783c651..3c066641d9f4 100644 --- a/node/core/dispute-coordinator/src/participation/mod.rs +++ b/node/core/dispute-coordinator/src/participation/mod.rs @@ -28,7 +28,7 @@ use futures_timer::Delay; use polkadot_node_primitives::{ValidationResult, APPROVAL_EXECUTION_TIMEOUT}; use polkadot_node_subsystem::{ messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage, CandidateValidationMessage}, - overseer, ActiveLeavesUpdate, RecoveryError, SubsystemContext, SubsystemSender, + overseer, ActiveLeavesUpdate, RecoveryError, }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; use polkadot_primitives::v2::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex}; diff --git a/node/core/dispute-coordinator/src/participation/queues/mod.rs b/node/core/dispute-coordinator/src/participation/queues/mod.rs index 3c7e2c25d55c..158bcfd2d3b6 100644 --- a/node/core/dispute-coordinator/src/participation/queues/mod.rs +++ b/node/core/dispute-coordinator/src/participation/queues/mod.rs @@ -20,7 +20,7 @@ use std::{ }; use futures::channel::oneshot; -use polkadot_node_subsystem::{messages::ChainApiMessage, overseer, SubsystemSender}; +use polkadot_node_subsystem::{messages::ChainApiMessage, overseer}; use polkadot_primitives::v2::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex}; use crate::{ diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index ff24c3282072..8bc5491b0216 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -32,7 +32,7 @@ use polkadot_node_subsystem::{ CandidateBackingMessage, ChainApiMessage, DisputeCoordinatorMessage, ProvisionableData, ProvisionerInherentData, ProvisionerMessage, }, - overseer, ActivatedLeaf, LeafStatus, PerLeafSpan, SubsystemSender, + overseer, ActivatedLeaf, LeafStatus, PerLeafSpan, }; use polkadot_node_subsystem_util::{ request_availability_cores, request_persisted_validation_data, JobSender, JobSubsystem, diff --git a/node/core/pvf-checker/src/lib.rs b/node/core/pvf-checker/src/lib.rs index e9eef19006eb..e5ace4c8f7ef 100644 --- a/node/core/pvf-checker/src/lib.rs +++ b/node/core/pvf-checker/src/lib.rs @@ -23,8 +23,8 @@ use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUn use polkadot_node_subsystem::{ messages::{CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage, RuntimeApiMessage}, - overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, - SubsystemError, SubsystemResult, SubsystemSender, + overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, + SubsystemResult, SubsystemSender, }; use polkadot_primitives::v2::{ BlockNumber, Hash, PvfCheckStatement, SessionIndex, ValidationCodeHash, ValidatorId, diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 914c5e961f26..38f01da13989 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -25,8 +25,7 @@ use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{RuntimeApiMessage, RuntimeApiRequest as Request}, - overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, - SubsystemResult, + overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, }; use polkadot_primitives::{ runtime_api::ParachainHost, diff --git a/node/malus/src/interceptor.rs b/node/malus/src/interceptor.rs index 80560aebfd03..7031b347fbdc 100644 --- a/node/malus/src/interceptor.rs +++ b/node/malus/src/interceptor.rs @@ -27,11 +27,9 @@ use std::{future::Future, pin::Pin}; /// Filter incoming and outgoing messages. pub trait MessageInterceptor: Send + Sync + Clone + 'static where - Sender: - overseer::SubsystemSender< - ::OutgoingMessages - > - + Clone + 'static, + Sender: overseer::SubsystemSender<::OutgoingMessages> + + Clone + + 'static, { /// The message type the original subsystem handles incoming. type Message: overseer::AssociateOutgoing + Send + 'static; @@ -50,7 +48,10 @@ where } /// Modify outgoing messages. - fn intercept_outgoing(&self, msg: ::OutgoingMessages) -> Option<::OutgoingMessages> { + fn intercept_outgoing( + &self, + msg: ::OutgoingMessages, + ) -> Option<::OutgoingMessages> { Some(msg) } } diff --git a/node/malus/src/variants/common.rs b/node/malus/src/variants/common.rs index b1f951ab948f..084cf73afa9e 100644 --- a/node/malus/src/variants/common.rs +++ b/node/malus/src/variants/common.rs @@ -131,10 +131,7 @@ where subsystem_sender: Sender, response_sender: oneshot::Sender>, ) where - Sender: overseer::CandidateValidationSenderTrait - + Clone - + Send - + 'static, + Sender: overseer::CandidateValidationSenderTrait + Clone + Send + 'static, { let _candidate_descriptor = candidate_descriptor.clone(); let mut subsystem_sender = subsystem_sender.clone(); @@ -199,10 +196,7 @@ fn create_validation_response( impl MessageInterceptor for ReplaceValidationResult where - Sender: overseer::CandidateValidationSenderTrait - + Clone - + Send - + 'static, + Sender: overseer::CandidateValidationSenderTrait + Clone + Send + 'static, Spawner: SpawnNamed + Clone + 'static, { type Message = CandidateValidationMessage; @@ -334,7 +328,10 @@ where } } - fn intercept_outgoing(&self, msg: ::OutgoingMessages) -> Option<::OutgoingMessages> { + fn intercept_outgoing( + &self, + msg: ::OutgoingMessages, + ) -> Option<::OutgoingMessages> { Some(msg) } } diff --git a/node/malus/src/variants/suggest_garbage_candidate.rs b/node/malus/src/variants/suggest_garbage_candidate.rs index dc51c765140a..c0813bdac6f5 100644 --- a/node/malus/src/variants/suggest_garbage_candidate.rs +++ b/node/malus/src/variants/suggest_garbage_candidate.rs @@ -71,10 +71,7 @@ struct NoteCandidate { impl MessageInterceptor for NoteCandidate where - Sender: overseer::CandidateBackingSenderTrait - + Clone - + Send - + 'static, + Sender: overseer::CandidateBackingSenderTrait + Clone + Send + 'static, Spawner: SpawnNamed + Clone + 'static, { type Message = CandidateBackingMessage; @@ -218,20 +215,21 @@ where } } - fn intercept_outgoing(&self, msg: overseer::CandidateBackingOutgoingMessages) -> Option { + fn intercept_outgoing( + &self, + msg: overseer::CandidateBackingOutgoingMessages, + ) -> Option { let msg = match msg { - overseer::CandidateBackingOutgoingMessages::CollatorProtocol(CollatorProtocolMessage::Seconded( - relay_parent, - statement, - )) => { + overseer::CandidateBackingOutgoingMessages::CollatorProtocol( + CollatorProtocolMessage::Seconded(relay_parent, statement), + ) => { // `parachain::collator-protocol: received an unexpected `CollationSeconded`: unknown statement statement=...` // TODO: Fix this error. We get this on colaltors because `malicious backing` creates a candidate that gets backed/included. // It is harmless for test parachain collators, but it will prevent cumulus based collators to make progress // as they wait for the relay chain to confirm the seconding of the collation. - overseer::CandidateBackingOutgoingMessages::CollatorProtocol(CollatorProtocolMessage::Seconded( - relay_parent, - statement, - )) + overseer::CandidateBackingOutgoingMessages::CollatorProtocol( + CollatorProtocolMessage::Seconded(relay_parent, statement), + ) }, msg => msg, }; diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index e51bb3595cf0..a4f19b741ad5 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -34,8 +34,7 @@ use polkadot_node_subsystem::{ ApprovalCheckResult, ApprovalDistributionMessage, ApprovalVotingMessage, AssignmentCheckResult, NetworkBridgeEvent, NetworkBridgeMessage, }, - overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, - SubsystemError, + overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, }; use polkadot_primitives::v2::{ BlockNumber, CandidateIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, @@ -792,7 +791,8 @@ impl State { ); match result { AssignmentCheckResult::Accepted => { - modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST).await; + modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST) + .await; entry.knowledge.known_messages.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { peer_knowledge.received.insert(message_subject.clone(), message_kind); @@ -820,7 +820,8 @@ impl State { ?peer_id, "Got an assignment too far in the future", ); - modify_reputation(ctx.sender(), peer_id, COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE).await; + modify_reputation(ctx.sender(), peer_id, COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE) + .await; return }, AssignmentCheckResult::Bad(error) => { @@ -1049,7 +1050,8 @@ impl State { ); match result { ApprovalCheckResult::Accepted => { - modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST).await; + modify_reputation(ctx.sender(), peer_id.clone(), BENEFIT_VALID_MESSAGE_FIRST) + .await; entry.knowledge.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { @@ -1568,7 +1570,11 @@ async fn adjust_required_routing_and_propagate, sender: &mut impl overseer::NetworkBridgeSenderTrait, ) { - event.focus() + event + .focus() .ok() .map(ApprovalDistributionMessage::from) .and_then(|msg| Some(sender.send_unbounded_message(msg))); - event.focus() + event + .focus() .ok() .map(BitfieldDistributionMessage::from) .and_then(|msg| Some(sender.send_unbounded_message(msg))); - event.focus() + event + .focus() .ok() .map(StatementDistributionMessage::from) .and_then(|msg| Some(sender.send_unbounded_message(msg))); @@ -1096,7 +1097,8 @@ async fn dispatch_validation_events_to_all( for event in events { sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; - sender.send_messages(event.focus().map(StatementDistributionMessage::from)) + sender + .send_messages(event.focus().map(StatementDistributionMessage::from)) .await; sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; } diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index e5ffbdb9c9d9..767d05aba357 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -41,7 +41,7 @@ use polkadot_node_subsystem::{ messages::{ CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, RuntimeApiMessage, }, - overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, + overseer, FromOverseer, OverseerSignal, PerLeafSpan, }; use polkadot_node_subsystem_util::{ metrics::{self, prometheus}, @@ -371,8 +371,7 @@ async fn distribute_collation( receipt: CandidateReceipt, pov: PoV, result_sender: Option>, -) -> Result<()> -{ +) -> Result<()> { let relay_parent = receipt.descriptor.relay_parent; // This collation is not in the active-leaves set. @@ -496,8 +495,7 @@ async fn determine_our_validators( core_index: CoreIndex, cores: usize, relay_parent: Hash, -) -> Result -{ +) -> Result { let session_index = runtime.get_session_index_for_child(ctx.sender(), relay_parent).await?; let info = &runtime .get_session_info_by_index(ctx.sender(), relay_parent, session_index) @@ -525,8 +523,7 @@ async fn determine_our_validators( /// Issue a `Declare` collation message to the given `peer`. #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] -async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) -{ +async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) { let declare_signature_payload = protocol_v1::declare_signature_payload(&state.local_peer_id); if let Some(para_id) = state.collating_on { @@ -547,8 +544,10 @@ async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) /// Issue a connection request to a set of validators and /// revoke the previous connection request. #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] -async fn connect_to_validators(ctx: &mut Context, validator_ids: Vec) -{ +async fn connect_to_validators( + ctx: &mut Context, + validator_ids: Vec, +) { // ignore address resolution failure // will reissue a new request on new collation let (failed, _) = oneshot::channel(); @@ -570,8 +569,7 @@ async fn advertise_collation( state: &mut State, relay_parent: Hash, peer: PeerId, -) -{ +) { let should_advertise = state .our_validators_groups .get(&relay_parent) @@ -630,8 +628,7 @@ async fn process_msg( runtime: &mut RuntimeInfo, state: &mut State, msg: CollatorProtocolMessage, -) -> Result<()> -{ +) -> Result<()> { use CollatorProtocolMessage::*; match msg { @@ -742,8 +739,7 @@ async fn handle_incoming_peer_message( state: &mut State, origin: PeerId, msg: protocol_v1::CollatorProtocolMessage, -) -> Result<()> -{ +) -> Result<()> { use protocol_v1::CollatorProtocolMessage::*; match msg { @@ -821,8 +817,7 @@ async fn handle_incoming_request( ctx: &mut Context, state: &mut State, req: IncomingRequest, -) -> Result<()> -{ +) -> Result<()> { let _span = state .span_per_relay_parent .get(&req.payload.relay_parent) @@ -896,8 +891,7 @@ async fn handle_peer_view_change( state: &mut State, peer_id: PeerId, view: View, -) -{ +) { let current = state.peer_views.entry(peer_id.clone()).or_default(); let added: Vec = view.difference(&*current).cloned().collect(); @@ -916,8 +910,7 @@ async fn handle_network_msg( runtime: &mut RuntimeInfo, state: &mut State, bridge_message: NetworkBridgeEvent, -) -> Result<()> -{ +) -> Result<()> { use NetworkBridgeEvent::*; match bridge_message { @@ -1008,8 +1001,7 @@ pub(crate) async fn run( collator_pair: CollatorPair, mut req_receiver: IncomingRequestReceiver, metrics: Metrics, -) -> std::result::Result<(), FatalError> -{ +) -> std::result::Result<(), FatalError> { use OverseerSignal::*; let mut state = State::new(local_peer_id, collator_pair, metrics); diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 2cd662d90107..4a3998f94a79 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -36,7 +36,7 @@ use polkadot_primitives::v2::CollatorPair; use polkadot_node_subsystem::{ errors::SubsystemError, messages::{CollatorProtocolMessage, NetworkBridgeMessage}, - overseer, SpawnedSubsystem, SubsystemContext, SubsystemSender, + overseer, SpawnedSubsystem, SubsystemSender, }; mod error; @@ -99,8 +99,7 @@ impl CollatorProtocolSubsystem { Self { protocol_side } } - async fn run(self, ctx: Context) -> std::result::Result<(), error::FatalError> - { + async fn run(self, ctx: Context) -> std::result::Result<(), error::FatalError> { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => validator_side::run(ctx, keystore, eviction_policy, metrics).await, @@ -123,8 +122,11 @@ impl CollatorProtocolSubsystem { } /// Modify the reputation of a peer based on its behavior. -async fn modify_reputation(sender: &mut impl overseer::CollatorProtocolSenderTrait, peer: PeerId, rep: Rep) -{ +async fn modify_reputation( + sender: &mut impl overseer::CollatorProtocolSenderTrait, + peer: PeerId, + rep: Rep, +) { gum::trace!( target: LOG_TARGET, rep = ?rep, diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index a79e6c6e14e3..3ebc253edb78 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -50,7 +50,7 @@ use polkadot_node_subsystem::{ CandidateBackingMessage, CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, NetworkBridgeMessage, RuntimeApiMessage, }, - overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, SubsystemSender, + overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemSender, }; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v2::{CandidateReceipt, CollatorId, Hash, Id as ParaId}; @@ -630,9 +630,9 @@ fn collator_peer_id( }) } -async fn disconnect_peer(sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_id: PeerId) -{ - sender.send_message(NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation)) +async fn disconnect_peer(sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_id: PeerId) { + sender + .send_message(NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation)) .await } @@ -642,8 +642,7 @@ async fn fetch_collation( state: &mut State, pc: PendingCollation, id: CollatorId, -) -{ +) { let (tx, rx) = oneshot::channel(); let PendingCollation { relay_parent, para_id, peer_id, .. } = pc; @@ -686,8 +685,7 @@ async fn report_collator( sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_data: &HashMap, id: CollatorId, -) -{ +) { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(sender, peer_id, COST_REPORT_BAD).await; } @@ -698,8 +696,7 @@ async fn note_good_collation( sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_data: &HashMap, id: CollatorId, -) -{ +) { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(sender, peer_id, BENEFIT_NOTIFY_GOOD).await; } @@ -711,15 +708,15 @@ async fn notify_collation_seconded( peer_id: PeerId, relay_parent: Hash, statement: SignedFullStatement, -) -{ +) { let wire_message = protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement.into()); - sender.send_message(NetworkBridgeMessage::SendCollationMessage( - vec![peer_id], - Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)), - )) - .await; + sender + .send_message(NetworkBridgeMessage::SendCollationMessage( + vec![peer_id], + Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)), + )) + .await; modify_reputation(sender, peer_id, BENEFIT_NOTIFY_GOOD).await; } @@ -751,8 +748,7 @@ async fn request_collation( para_id: ParaId, peer_id: PeerId, result: oneshot::Sender<(CandidateReceipt, PoV)>, -) -{ +) { if !state.view.contains(&relay_parent) { gum::debug!( target: LOG_TARGET, @@ -803,11 +799,12 @@ async fn request_collation( "Requesting collation", ); - sender.send_message(NetworkBridgeMessage::SendRequests( - vec![requests], - IfDisconnected::ImmediateError, - )) - .await; + sender + .send_message(NetworkBridgeMessage::SendRequests( + vec![requests], + IfDisconnected::ImmediateError, + )) + .await; } /// Networking message has been received. @@ -817,8 +814,7 @@ async fn process_incoming_peer_message( state: &mut State, origin: PeerId, msg: protocol_v1::CollatorProtocolMessage, -) -{ +) { use protocol_v1::CollatorProtocolMessage::*; use sp_runtime::traits::AppVerify; match msg { @@ -949,7 +945,8 @@ async fn process_incoming_peer_message( collations.status = CollationStatus::Fetching; collations.waiting_collation = Some(id.clone()); - fetch_collation(ctx.sender(), state, pending_collation.clone(), id).await; + fetch_collation(ctx.sender(), state, pending_collation.clone(), id) + .await; }, CollationStatus::Seconded => { gum::trace!( @@ -1004,8 +1001,7 @@ async fn handle_our_view_change( state: &mut State, keystore: &SyncCryptoStorePtr, view: OurView, -) -> Result<()> -{ +) -> Result<()> { let old_view = std::mem::replace(&mut state.view, view); let added: HashMap> = state @@ -1061,8 +1057,7 @@ async fn handle_network_msg( state: &mut State, keystore: &SyncCryptoStorePtr, bridge_message: NetworkBridgeEvent, -) -> Result<()> -{ +) -> Result<()> { use NetworkBridgeEvent::*; match bridge_message { @@ -1098,8 +1093,7 @@ async fn process_msg( keystore: &SyncCryptoStorePtr, msg: CollatorProtocolMessage, state: &mut State, -) -{ +) { use CollatorProtocolMessage::*; let _timer = state.metrics.time_process_msg(); @@ -1199,8 +1193,7 @@ pub(crate) async fn run( keystore: SyncCryptoStorePtr, eviction_policy: crate::CollatorEvictionPolicy, metrics: Metrics, -) -> std::result::Result<(), crate::error::FatalError> -{ +) -> std::result::Result<(), crate::error::FatalError> { let mut state = State { metrics, ..Default::default() }; let next_inactivity_stream = infinite_stream(ACTIVITY_POLL); @@ -1291,8 +1284,7 @@ async fn dequeue_next_collation_and_fetch( relay_parent: Hash, // The collator we tried to fetch from last. previous_fetch: CollatorId, -) -{ +) { if let Some((next, id)) = state .collations_per_relay_parent .get_mut(&relay_parent) @@ -1321,8 +1313,7 @@ async fn handle_collation_fetched_result( ctx: &mut Context, state: &mut State, (mut collation_event, res): PendingCollationFetch, -) -{ +) { // If no prior collation for this relay parent has been seconded, then // memorize the `collation_event` for that `relay_parent`, such that we may // notify the collator of their successful second backing @@ -1360,12 +1351,13 @@ async fn handle_collation_fetched_result( if let Entry::Vacant(entry) = state.pending_candidates.entry(relay_parent) { collation_event.1.commitments_hash = Some(candidate_receipt.commitments_hash); - ctx.sender().send_message(CandidateBackingMessage::Second( - relay_parent.clone(), - candidate_receipt, - pov, - )) - .await; + ctx.sender() + .send_message(CandidateBackingMessage::Second( + relay_parent.clone(), + candidate_receipt, + pov, + )) + .await; entry.insert(collation_event); } else { @@ -1385,8 +1377,7 @@ async fn disconnect_inactive_peers( sender: &mut impl overseer::CollatorProtocolSenderTrait, eviction_policy: &crate::CollatorEvictionPolicy, peers: &HashMap, -) -{ +) { for (peer, peer_data) in peers { if peer_data.is_inactive(&eviction_policy) { gum::trace!(target: LOG_TARGET, "Disconnecting inactive peer"); diff --git a/node/network/dispute-distribution/src/lib.rs b/node/network/dispute-distribution/src/lib.rs index 93a4536f4d28..3b195aa7b3e7 100644 --- a/node/network/dispute-distribution/src/lib.rs +++ b/node/network/dispute-distribution/src/lib.rs @@ -33,7 +33,7 @@ use polkadot_node_network_protocol::request_response::{incoming::IncomingRequest use polkadot_node_primitives::DISPUTE_WINDOW; use polkadot_node_subsystem::{ messages::DisputeDistributionMessage, overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, - SubsystemContext, SubsystemError, + SubsystemError, }; use polkadot_node_subsystem_util::{runtime, runtime::RuntimeInfo}; diff --git a/node/network/dispute-distribution/src/receiver/mod.rs b/node/network/dispute-distribution/src/receiver/mod.rs index 82554569aba6..e061e67f4301 100644 --- a/node/network/dispute-distribution/src/receiver/mod.rs +++ b/node/network/dispute-distribution/src/receiver/mod.rs @@ -41,7 +41,7 @@ use polkadot_node_network_protocol::{ use polkadot_node_primitives::DISPUTE_WINDOW; use polkadot_node_subsystem::{ messages::{DisputeCoordinatorMessage, ImportStatementsResult}, - overseer, SubsystemSender, + overseer, }; use polkadot_node_subsystem_util::{runtime, runtime::RuntimeInfo}; diff --git a/node/network/dispute-distribution/src/sender/mod.rs b/node/network/dispute-distribution/src/sender/mod.rs index 975c69b0fe30..150e79eda108 100644 --- a/node/network/dispute-distribution/src/sender/mod.rs +++ b/node/network/dispute-distribution/src/sender/mod.rs @@ -20,9 +20,7 @@ use futures::channel::{mpsc, oneshot}; use polkadot_node_network_protocol::request_response::v1::DisputeRequest; use polkadot_node_primitives::{CandidateVotes, DisputeMessage, SignedDisputeStatement}; -use polkadot_node_subsystem::{ - messages::DisputeCoordinatorMessage, overseer, ActiveLeavesUpdate, SubsystemContext, -}; +use polkadot_node_subsystem::{messages::DisputeCoordinatorMessage, overseer, ActiveLeavesUpdate}; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::v2::{CandidateHash, DisputeStatement, Hash, SessionIndex}; diff --git a/node/network/dispute-distribution/src/sender/send_task.rs b/node/network/dispute-distribution/src/sender/send_task.rs index d5f3a9feb2ec..03ce192697bb 100644 --- a/node/network/dispute-distribution/src/sender/send_task.rs +++ b/node/network/dispute-distribution/src/sender/send_task.rs @@ -26,7 +26,7 @@ use polkadot_node_network_protocol::{ }, IfDisconnected, }; -use polkadot_node_subsystem::{messages::NetworkBridgeMessage, overseer, SubsystemContext}; +use polkadot_node_subsystem::{messages::NetworkBridgeMessage, overseer}; use polkadot_node_subsystem_util::{metrics, runtime::RuntimeInfo}; use polkadot_primitives::v2::{ AuthorityDiscoveryId, CandidateHash, Hash, SessionIndex, ValidatorIndex, diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 0d026cf32bb4..bd323b3fe96e 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -48,8 +48,7 @@ use polkadot_node_subsystem::{ GossipSupportMessage, NetworkBridgeEvent, NetworkBridgeMessage, RuntimeApiMessage, RuntimeApiRequest, }, - overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, - SubsystemError, + overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, }; use polkadot_node_subsystem_util as util; use polkadot_primitives::v2::{ diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index fbdeafa86cf4..cfd19713a84a 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -42,7 +42,7 @@ use polkadot_node_subsystem::{ StatementDistributionMessage, }, overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, PerLeafSpan, SpawnedSubsystem, - SubsystemContext, SubsystemError, + SubsystemError, }; use polkadot_primitives::v2::{ AuthorityDiscoveryId, CandidateHash, CommittedCandidateReceipt, CompactStatement, Hash, @@ -1158,7 +1158,11 @@ async fn send_statements( } } -async fn report_peer(sender: &mut impl overseer::StatementDistributionSenderTrait, peer: PeerId, rep: Rep) { +async fn report_peer( + sender: &mut impl overseer::StatementDistributionSenderTrait, + peer: PeerId, + rep: Rep, +) { sender.send_message(NetworkBridgeMessage::ReportPeer(peer, rep)).await } diff --git a/node/service/src/overseer.rs b/node/service/src/overseer.rs index 392d042dd4b9..bb3d9e840f1c 100644 --- a/node/service/src/overseer.rs +++ b/node/service/src/overseer.rs @@ -24,17 +24,17 @@ use polkadot_node_core_chain_selection::Config as ChainSelectionConfig; use polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig; use polkadot_node_core_provisioner::ProvisionerConfig; use polkadot_node_network_protocol::request_response::{v1 as request_v1, IncomingRequestReceiver}; +use polkadot_node_subsystem_types::messages::{BitfieldSigningMessage, ProvisionerMessage}; #[cfg(any(feature = "malus", test))] pub use polkadot_overseer::{ dummy::{dummy_overseer_builder, DummySubsystem}, HeadSupportsParachains, }; use polkadot_overseer::{ - metrics::Metrics as OverseerMetrics, BlockInfo, InitializedOverseerBuilder, MetricsTrait, - Overseer, OverseerConnector, OverseerHandle, OverseerSubsystemContext, + gen::SubsystemContext, metrics::Metrics as OverseerMetrics, BlockInfo, + InitializedOverseerBuilder, MetricsTrait, Overseer, OverseerConnector, OverseerHandle, + OverseerSubsystemContext, }; -use polkadot_node_subsystem_types::messages::{BitfieldSigningMessage, ProvisionerMessage}; -use polkadot_overseer::gen::SubsystemContext; use polkadot_primitives::runtime_api::ParachainHost; use sc_authority_discovery::Service as AuthorityDiscoveryService; @@ -154,9 +154,15 @@ pub fn prepared_overseer_builder<'a, Spawner, RuntimeClient>( StatementDistributionSubsystem, AvailabilityDistributionSubsystem, AvailabilityRecoverySubsystem, - BitfieldSigningSubsystem as SubsystemContext>::Sender>, + BitfieldSigningSubsystem< + Spawner, + as SubsystemContext>::Sender, + >, BitfieldDistributionSubsystem, - ProvisionerSubsystem as SubsystemContext>::Sender>, + ProvisionerSubsystem< + Spawner, + as SubsystemContext>::Sender, + >, RuntimeApiSubsystem, AvailabilityStoreSubsystem, NetworkBridgeSubsystem< From d3956ae1d0160a01dd3c35440005ace43f83177f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 6 May 2022 15:55:21 +0200 Subject: [PATCH 099/139] fix interceptor --- node/malus/src/interceptor.rs | 116 ++++++++++++++----------- node/subsystem-test-helpers/src/lib.rs | 2 +- 2 files changed, 68 insertions(+), 50 deletions(-) diff --git a/node/malus/src/interceptor.rs b/node/malus/src/interceptor.rs index 7031b347fbdc..bf31bf4359f2 100644 --- a/node/malus/src/interceptor.rs +++ b/node/malus/src/interceptor.rs @@ -21,7 +21,7 @@ //! messages on the overseer level. use polkadot_node_subsystem::*; -pub use polkadot_node_subsystem::{messages, overseer, FromOverseer}; +pub use polkadot_node_subsystem::{messages, messages::*, overseer, FromOverseer}; use std::{future::Future, pin::Pin}; /// Filter incoming and outgoing messages. @@ -64,33 +64,46 @@ pub struct InterceptedSender { } #[async_trait::async_trait] -impl overseer::SubsystemSender for InterceptedSender +impl overseer::SubsystemSender for InterceptedSender where - M: overseer::AssociateOutgoing, - ::OutgoingMessages: From, - Sender: overseer::SubsystemSender - + overseer::SubsystemSender<::OutgoingMessages> - + overseer::SubsystemSender<<>::Message as overseer::AssociateOutgoing>::OutgoingMessages>, + OutgoingMessage: overseer::AssociateOutgoing + Send + 'static, + Sender: overseer::SubsystemSender + + overseer::SubsystemSender< + < + >::Message as overseer::AssociateOutgoing + >::OutgoingMessages + >, Fil: MessageInterceptor, + >::Message: overseer::AssociateOutgoing, + < + >::Message as overseer::AssociateOutgoing + >::OutgoingMessages: + From, { - async fn send_message(&mut self, msg: M) { - if let Some(msg) = self.message_filter.intercept_outgoing(<::OutgoingMessages>::from(msg)) { + async fn send_message(&mut self, msg: OutgoingMessage) { + let msg = < + <>::Message as overseer::AssociateOutgoing + >::OutgoingMessages as From>::from(msg); + if let Some(msg) = self.message_filter.intercept_outgoing(msg) { self.inner.send_message(msg).await; } } async fn send_messages(&mut self, msgs: T) where - T: IntoIterator + Send, + T: IntoIterator + Send, T::IntoIter: Send, { for msg in msgs { - self.send_message(<::OutgoingMessages>::from(msg)).await; + self.send_message(msg).await; } } - fn send_unbounded_message(&mut self, msg: M) { - if let Some(msg) = self.message_filter.intercept_outgoing(<::OutgoingMessages>::from(msg)) { + fn send_unbounded_message(&mut self, msg: OutgoingMessage) { + let msg = < + <>::Message as overseer::AssociateOutgoing + >::OutgoingMessages as From>::from(msg); + if let Some(msg) = self.message_filter.intercept_outgoing(msg) { self.inner.send_unbounded_message(msg); } } @@ -99,16 +112,16 @@ where /// A subsystem context, that filters the outgoing messages. pub struct InterceptedContext where - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor<::Sender>, - ::Sender: overseer::SubsystemSender< - ::Sender>>::Message, - >, - ::Sender: overseer::SubsystemSender< + ::Sender: + overseer::SubsystemSender< < - ::Sender> - >::Message as AssociateOutgoing - >::OutgoingMessages>, + < + Fil as MessageInterceptor<::Sender> + >::Message as overseer::AssociateOutgoing + >::OutgoingMessages, + >, { inner: Context, message_filter: Fil, @@ -117,15 +130,12 @@ where impl InterceptedContext where - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext, Fil: MessageInterceptor< ::Sender, Message = ::Message, >, - ::Message: AssociateOutgoing, - ::Sender: overseer::SubsystemSender< - ::Sender>>::Message, - >, + ::Message: overseer::AssociateOutgoing, ::Sender: overseer::SubsystemSender< <::Message as overseer::AssociateOutgoing>::OutgoingMessages > @@ -142,28 +152,27 @@ where #[async_trait::async_trait] impl overseer::SubsystemContext for InterceptedContext where - Context: overseer::SubsystemContext, - Fil: MessageInterceptor< - ::Sender, - Message = ::Message, - >, + Context: overseer::SubsystemContext, + ::Message: + overseer::AssociateOutgoing, ::Sender: overseer::SubsystemSender< <::Message as overseer::AssociateOutgoing>::OutgoingMessages >, - ::Sender: + InterceptedSender<::Sender, Fil>: overseer::SubsystemSender< - ::Sender>>::Message, + <::Message as overseer::AssociateOutgoing>::OutgoingMessages >, - ::Message: overseer::AssociateOutgoing, - ::Sender: - overseer::SubsystemSender<<::Message as overseer::AssociateOutgoing>::OutgoingMessages>, + Fil: MessageInterceptor< + ::Sender, + Message = ::Message, + >, { type Message = ::Message; type Sender = InterceptedSender<::Sender, Fil>; type Error = SubsystemError; type OutgoingMessages = <::Message as overseer::AssociateOutgoing>::OutgoingMessages; - type Signal = ::Signal; + type Signal = OverseerSignal; async fn try_recv(&mut self) -> Result>, ()> { loop { @@ -223,19 +232,28 @@ impl InterceptedSubsystem { impl overseer::Subsystem for InterceptedSubsystem where - Context: overseer::SubsystemContext + Sync + Send, - Sub: overseer::Subsystem, SubsystemError>, - InterceptedContext: overseer::SubsystemContext, - Interceptor: MessageInterceptor< - ::Sender, - Message = ::Message, - >, - ::Sender: overseer::SubsystemSender< - ::Sender>>::Message, - >, - ::Message: AssociateOutgoing, + Context: + overseer::SubsystemContext + Sync + Send, + InterceptedContext: + overseer::SubsystemContext, + Sub: + overseer::Subsystem, SubsystemError>, + Interceptor: + MessageInterceptor< + ::Sender, + Message = ::Message, + >, + ::Sender: + overseer::SubsystemSender< + ::Sender>>::Message, + >, + ::Message: + overseer::AssociateOutgoing, ::Sender: - overseer::SubsystemSender<<::Message as overseer::AssociateOutgoing>::OutgoingMessages>, + overseer::SubsystemSender< + <::Message as overseer::AssociateOutgoing + >::OutgoingMessages + >, { fn start(self, ctx: Context) -> SpawnedSubsystem { let ctx = InterceptedContext::new(ctx, self.message_interceptor); diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index 5994c24b9b4a..624b0af73d55 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -20,7 +20,7 @@ use polkadot_node_subsystem::{ messages::AllMessages, overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, - SubsystemContext, SubsystemError, SubsystemResult, + SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::TimeoutExt; From 99c3c776d8c1fd1b690e00447e6401a75fce8bd2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 6 May 2022 17:04:04 +0200 Subject: [PATCH 100/139] fix malus compilation FUCK YEAH --- node/malus/src/interceptor.rs | 8 ++++---- node/malus/src/tests.rs | 4 ++-- node/malus/src/variants/common.rs | 4 ++-- node/malus/src/variants/suggest_garbage_candidate.rs | 4 ++-- .../proc-macro/src/impl_subsystem_ctx_sender.rs | 5 +++++ 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/node/malus/src/interceptor.rs b/node/malus/src/interceptor.rs index bf31bf4359f2..19889b42e32a 100644 --- a/node/malus/src/interceptor.rs +++ b/node/malus/src/interceptor.rs @@ -243,10 +243,10 @@ where ::Sender, Message = ::Message, >, - ::Sender: - overseer::SubsystemSender< - ::Sender>>::Message, - >, + // ::Sender: + // overseer::SubsystemSender< + // ::Sender>>::Message, + // >, ::Message: overseer::AssociateOutgoing, ::Sender: diff --git a/node/malus/src/tests.rs b/node/malus/src/tests.rs index eb89acc0667d..6ce65fd9fc3b 100644 --- a/node/malus/src/tests.rs +++ b/node/malus/src/tests.rs @@ -66,8 +66,8 @@ async fn overseer_send>(overseer: &mut TestSubsystemContext fn launch_harness(test_gen: G) where F: Future> + Send, - M: AssociateOutgoing + Into<::OutgoingMessages> + std::fmt::Debug + Send + 'static, - ::OutgoingMessages: From, + M: AssociateOutgoing + std::fmt::Debug + Send + 'static, + // ::OutgoingMessages: From, Sub: Subsystem, SubsystemError>, G: Fn(TestSubsystemContextHandle) -> (F, Sub), { diff --git a/node/malus/src/variants/common.rs b/node/malus/src/variants/common.rs index 084cf73afa9e..138a91b759ab 100644 --- a/node/malus/src/variants/common.rs +++ b/node/malus/src/variants/common.rs @@ -330,8 +330,8 @@ where fn intercept_outgoing( &self, - msg: ::OutgoingMessages, - ) -> Option<::OutgoingMessages> { + msg: overseer::CandidateValidationOutgoingMessages, + ) -> Option { Some(msg) } } diff --git a/node/malus/src/variants/suggest_garbage_candidate.rs b/node/malus/src/variants/suggest_garbage_candidate.rs index c0813bdac6f5..ef987278decb 100644 --- a/node/malus/src/variants/suggest_garbage_candidate.rs +++ b/node/malus/src/variants/suggest_garbage_candidate.rs @@ -220,14 +220,14 @@ where msg: overseer::CandidateBackingOutgoingMessages, ) -> Option { let msg = match msg { - overseer::CandidateBackingOutgoingMessages::CollatorProtocol( + overseer::CandidateBackingOutgoingMessages::CollatorProtocolMessage( CollatorProtocolMessage::Seconded(relay_parent, statement), ) => { // `parachain::collator-protocol: received an unexpected `CollationSeconded`: unknown statement statement=...` // TODO: Fix this error. We get this on colaltors because `malicious backing` creates a candidate that gets backed/included. // It is harmless for test parachain collators, but it will prevent cumulus based collators to make progress // as they wait for the relay chain to confirm the seconding of the collation. - overseer::CandidateBackingOutgoingMessages::CollatorProtocol( + overseer::CandidateBackingOutgoingMessages::CollatorProtocolMessage( CollatorProtocolMessage::Seconded(relay_parent, statement), ) }, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index bbcd11d0b67d..15c1303ff0d8 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -201,6 +201,11 @@ fn to_variant(path: &Path, span: Span) -> Result { Ok(ident) } + +/// Converts the outgoing message types to variants. +/// +/// TODO: Commonly this is `${X}Message` becomes `${X}OutgoingMessages::${X}Message` +/// where for `AllMessages` it would be `AllMessages::${X}`. fn to_variants(message_types: &[Path], span: Span) -> Result> { let variants: Vec<_> = Result::from_iter(message_types.into_iter().map(|path| to_variant(path, span.clone())))?; From ea5540ae66a996d515b50c787179b6f044ef5f1f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 6 May 2022 17:50:33 +0200 Subject: [PATCH 101/139] fix examples --- node/overseer/overseer-gen/Cargo.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 8a8d1a61ca20..e8c2b569a023 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -4,6 +4,7 @@ version = "0.9.19" authors = ["Parity Technologies "] edition = "2021" description = "Generate an overseer including builder pattern and message wrapper from a single struct." +autoexamples = false [dependencies] gum = { package = "tracing-gum", path = "../../gum" } @@ -22,6 +23,16 @@ pin-project = "1.0" trybuild = "1.0.61" rustversion = "1.0.6" + + +[[example]] +name = "duo" +crate-type = ["bin"] + +[[example]] +name = "solo" +crate-type = ["bin"] + [features] default = [] expand = ["polkadot-overseer-gen-proc-macro/expand"] From 48f43642677ef5538d1edffebaa2f46416214958 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 6 May 2022 17:53:31 +0200 Subject: [PATCH 102/139] remove foo --- node/overseer/overseer-gen/examples/duo.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs index b50c671eddf0..66c33ddcd0a5 100644 --- a/node/overseer/overseer-gen/examples/duo.rs +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -1,6 +1,5 @@ //! A dummy to be used with cargo expand -use polkadot_node_network_protocol::WrongVariant; use polkadot_overseer_gen::{self as overseer, SpawnNamed, *}; use std::collections::HashMap; mod misc; From 026b20d3ed9dfd0a4bb3a5bf8bfa2f71f4223e9e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 6 May 2022 17:53:46 +0200 Subject: [PATCH 103/139] crazy unit type impl --- .../src/impl_subsystem_ctx_sender.rs | 219 ++++++++++-------- 1 file changed, 122 insertions(+), 97 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index 15c1303ff0d8..49d65667e56c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -16,7 +16,7 @@ use proc_macro2::TokenStream; use quote::quote; -use syn::{Ident, Path, Result}; +use syn::{Ident, Path, Result, Type}; use petgraph::{ dot::{self, Dot}, @@ -201,7 +201,6 @@ fn to_variant(path: &Path, span: Span) -> Result { Ok(ident) } - /// Converts the outgoing message types to variants. /// /// TODO: Commonly this is `${X}Message` becomes `${X}OutgoingMessages::${X}Message` @@ -395,6 +394,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( #( + #support_crate ::SubsystemSender< #outgoing > )* + + #support_crate ::SubsystemSender< () > + Send + 'static }; @@ -415,8 +415,9 @@ pub(crate) fn impl_per_subsystem_helper_traits( #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, #all_messages_wrapper: From< #outgoing_wrapper >, #all_messages_wrapper: From< #consumes >, + #all_messages_wrapper: From< () >, #outgoing_wrapper: #( From< #outgoing > )+*, - + #outgoing_wrapper: From< () >, }; ts.extend(quote! { @@ -457,116 +458,140 @@ pub(crate) fn impl_per_subsystem_helper_traits( }); // impl the subsystem context trait - ts.extend(quote! { - #[#support_crate ::async_trait] - impl #support_crate ::SubsystemContext for #subsystem_ctx_name < #consumes > - where - #where_clause - { - type Message = #consumes; - type Signal = #signal; - type OutgoingMessages = #outgoing_wrapper; - type Sender = #subsystem_sender_name < #outgoing_wrapper >; - type Error = #error_ty; + let impl_subsystem_ctx_trait = |consumes: Type, + outgoing: &[Type], + outgoing_wrapper: Type| + -> TokenStream { + let where_clause = quote! { + #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, + #all_messages_wrapper: From< #outgoing_wrapper >, + #all_messages_wrapper: From< #consumes >, + #outgoing_wrapper: #( From< #outgoing > )+*, + }; - async fn try_recv(&mut self) -> ::std::result::Result>, ()> { - match #support_crate ::poll!(self.recv()) { - #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), - #support_crate ::Poll::Pending => Ok(None), + quote! { + #[#support_crate ::async_trait] + impl #support_crate ::SubsystemContext for #subsystem_ctx_name < #consumes > + where + #where_clause + { + type Message = #consumes; + type Signal = #signal; + type OutgoingMessages = #outgoing_wrapper; + type Sender = #subsystem_sender_name < #outgoing_wrapper >; + type Error = #error_ty; + + async fn try_recv(&mut self) -> ::std::result::Result>, ()> { + match #support_crate ::poll!(self.recv()) { + #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), + #support_crate ::Poll::Pending => Ok(None), + } } - } - async fn recv(&mut self) -> ::std::result::Result, #error_ty> { - loop { - // If we have a message pending an overseer signal, we only poll for signals - // in the meantime. - if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { - if needs_signals_received <= self.signals_received.load() { - return Ok( #support_crate ::FromOverseer::Communication { msg }); - } else { - self.pending_incoming = Some((needs_signals_received, msg)); - - // wait for next signal. - let signal = self.signals.next().await - .ok_or(#support_crate ::OverseerError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; + async fn recv(&mut self) -> ::std::result::Result, #error_ty> { + loop { + // If we have a message pending an overseer signal, we only poll for signals + // in the meantime. + if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { + if needs_signals_received <= self.signals_received.load() { + return Ok( #support_crate ::FromOverseer::Communication { msg }); + } else { + self.pending_incoming = Some((needs_signals_received, msg)); - self.signals_received.inc(); - return Ok( #support_crate ::FromOverseer::Signal(signal)) - } - } + // wait for next signal. + let signal = self.signals.next().await + .ok_or(#support_crate ::OverseerError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; - let mut await_message = self.messages.next().fuse(); - let mut await_signal = self.signals.next().fuse(); - let signals_received = self.signals_received.load(); - let pending_incoming = &mut self.pending_incoming; - - // Otherwise, wait for the next signal or incoming message. - let from_overseer = #support_crate ::futures::select_biased! { - signal = await_signal => { - let signal = signal - .ok_or( #support_crate ::OverseerError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; - - #support_crate ::FromOverseer::Signal(signal) + self.signals_received.inc(); + return Ok( #support_crate ::FromOverseer::Signal(signal)) + } } - msg = await_message => { - let packet = msg - .ok_or( #support_crate ::OverseerError::Context( - "Message channel is terminated and empty." - .to_owned() - ))?; - - if packet.signals_received > signals_received { - // wait until we've received enough signals to return this message. - *pending_incoming = Some((packet.signals_received, packet.message)); - continue; - } else { - // we know enough to return this message. - #support_crate ::FromOverseer::Communication { msg: packet.message} + + let mut await_message = self.messages.next().fuse(); + let mut await_signal = self.signals.next().fuse(); + let signals_received = self.signals_received.load(); + let pending_incoming = &mut self.pending_incoming; + + // Otherwise, wait for the next signal or incoming message. + let from_overseer = #support_crate ::futures::select_biased! { + signal = await_signal => { + let signal = signal + .ok_or( #support_crate ::OverseerError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; + + #support_crate ::FromOverseer::Signal(signal) } + msg = await_message => { + let packet = msg + .ok_or( #support_crate ::OverseerError::Context( + "Message channel is terminated and empty." + .to_owned() + ))?; + + if packet.signals_received > signals_received { + // wait until we've received enough signals to return this message. + *pending_incoming = Some((packet.signals_received, packet.message)); + continue; + } else { + // we know enough to return this message. + #support_crate ::FromOverseer::Communication { msg: packet.message} + } + } + }; + + if let #support_crate ::FromOverseer::Signal(_) = from_overseer { + self.signals_received.inc(); } - }; - if let #support_crate ::FromOverseer::Signal(_) = from_overseer { - self.signals_received.inc(); + return Ok(from_overseer); } - - return Ok(from_overseer); } - } - fn sender(&mut self) -> &mut Self::Sender { - &mut self.to_subsystems - } + fn sender(&mut self) -> &mut Self::Sender { + &mut self.to_subsystems + } - fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - -> ::std::result::Result<(), #error_ty> - { - self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { - name, - subsystem: Some(self.name()), - s, - }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; - Ok(()) - } + fn spawn(&mut self, name: &'static str, s: Pin + Send>>) + -> ::std::result::Result<(), #error_ty> + { + self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { + name, + subsystem: Some(self.name()), + s, + }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; + Ok(()) + } - fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - -> ::std::result::Result<(), #error_ty> - { - self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { - name, - subsystem: Some(self.name()), - s, - }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; - Ok(()) + fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + -> ::std::result::Result<(), #error_ty> + { + self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { + name, + subsystem: Some(self.name()), + s, + }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; + Ok(()) + } } } - }); + }; + + ts.extend(impl_subsystem_ctx_trait( + parse_quote! { #consumes }, + &Vec::from_iter(outgoing.iter().map(|path| { + parse_quote! { #path } + })), + parse_quote! { #outgoing_wrapper }, + )); + + let empty_tuple: Type = parse_quote! { () }; + ts.extend(impl_subsystem_ctx_trait(empty_tuple.clone(), &[], empty_tuple)); + ts } From 224561c9fb5136b77940a15a000bb8a326344de8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 11:33:28 +0200 Subject: [PATCH 104/139] test fixes --- node/collation-generation/src/tests.rs | 16 +++++++++------- .../src/requester/fetch_task/tests.rs | 6 +++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/node/collation-generation/src/tests.rs b/node/collation-generation/src/tests.rs index 90e241877147..563e3cfdce02 100644 --- a/node/collation-generation/src/tests.rs +++ b/node/collation-generation/src/tests.rs @@ -328,7 +328,7 @@ mod handle_new_activations { }; assert_eq!(sent_messages.len(), 1); - match &sent_messages[0] { + match AllMessages::from(sent_messages.pop().unwrap()) { AllMessages::CollatorProtocol(CollatorProtocolMessage::DistributeCollation( CandidateReceipt { descriptor, .. }, _pov, @@ -356,7 +356,7 @@ mod handle_new_activations { expect_descriptor.erasure_root = descriptor.erasure_root.clone(); expect_descriptor }; - assert_eq!(descriptor, &expect_descriptor); + assert_eq!(descriptor, expect_descriptor); }, _ => panic!("received wrong message type"), } @@ -470,11 +470,13 @@ mod handle_new_activations { assert_eq!(sent_messages.len(), 1); match &sent_messages[0] { - AllMessages::CollatorProtocol(CollatorProtocolMessage::DistributeCollation( - CandidateReceipt { descriptor, .. }, - _pov, - .., - )) => { + overseer::CollationGenerationOutgoingMessages::CollatorProtocolMessage( + CollatorProtocolMessage::DistributeCollation( + CandidateReceipt { descriptor, .. }, + _pov, + .., + ), + ) => { assert_eq!(expect_validation_code_hash, descriptor.validation_code_hash); }, _ => panic!("received wrong message type"), diff --git a/node/network/availability-distribution/src/requester/fetch_task/tests.rs b/node/network/availability-distribution/src/requester/fetch_task/tests.rs index dea0d187f2d8..3030cb877ec7 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/tests.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/tests.rs @@ -227,7 +227,11 @@ impl TestRun { /// Returns true, if after processing of the given message it would be OK for the stream to /// end. - async fn handle_message(&self, msg: AllMessages) -> bool { + async fn handle_message( + &self, + msg: overseer::AvailabilityDistributionOutgoingMessages, + ) -> bool { + let msg = AllMessages::from(msg); match msg { AllMessages::NetworkBridge(NetworkBridgeMessage::SendRequests( reqs, From 3dba834462d4c69713cfd6dcf86af18fb6dee3de Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 11:33:35 +0200 Subject: [PATCH 105/139] example fix --- node/overseer/examples/minimal-example.rs | 4 ++-- node/overseer/overseer-gen/examples/solo.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 28c66d57ece7..036594306d63 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -85,8 +85,8 @@ impl Subsystem1 { } #[overseer::contextbounds(CandidateBacking, error = SubsystemError, prefix = self::overseer)] -impl Subsystem1 { - fn start(self, ctx: Context) -> SpawnedSubsystem { +impl Subsystem1 { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { Self::run(ctx).await; Ok(()) diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index 9d0a6fefe839..adb6869a63ee 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -1,3 +1,5 @@ +#![allow(unused_assignments)] + //! A minimal demo to be used with cargo expand. use overseer::SubsystemSender as _; From b410971e06c8d549d6f5f37053b1f7bfb881ad87 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 11:34:02 +0200 Subject: [PATCH 106/139] avoid duplicate code generation --- .../src/impl_subsystem_ctx_sender.rs | 296 ++++++++++-------- 1 file changed, 161 insertions(+), 135 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index 49d65667e56c..3b428c1c2858 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -58,6 +58,8 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { let span = overseer_name.span(); let all_messages_wrapper = &info.message_wrapper; let support_crate = info.support_crate_name(); + let signal_ty = &info.extern_signal_ty; + let error_ty = &info.extern_error_ty; // create a directed graph with all the subsystems as nodes and the messages as edges // key is always the message path, values are node indices in the graph and the subsystem generic identifier @@ -188,6 +190,21 @@ pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { ts.extend(impl_wrapper_enum(&outgoing_wrapper, ssf.messages_to_send.as_slice())?); } + + // impl the emtpy tuple handling for tests + let empty_tuple: Type = parse_quote! { () }; + ts.extend(impl_subsystem_context_trait_for( + empty_tuple.clone(), + &[], + empty_tuple, + all_messages_wrapper, + subsystem_ctx_name, + subsystem_sender_name, + support_crate, + signal_ty, + error_ty, + )); + Ok(ts) } @@ -370,6 +387,137 @@ pub(crate) fn impl_associate_outgoing_messages( } } +pub(crate) fn impl_subsystem_context_trait_for( + consumes: Type, + outgoing: &[Type], + outgoing_wrapper: Type, + all_messages_wrapper: &Ident, + subsystem_ctx_name: &Ident, + subsystem_sender_name: &Ident, + support_crate: &Path, + signal: &Path, + error_ty: &Path, +) -> TokenStream { + // impl the subsystem context trait + let where_clause = quote! { + #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, + #all_messages_wrapper: From< #outgoing_wrapper >, + #all_messages_wrapper: From< #consumes >, + #outgoing_wrapper: #( From< #outgoing > )+*, + }; + + quote! { + #[#support_crate ::async_trait] + impl #support_crate ::SubsystemContext for #subsystem_ctx_name < #consumes > + where + #where_clause + { + type Message = #consumes; + type Signal = #signal; + type OutgoingMessages = #outgoing_wrapper; + type Sender = #subsystem_sender_name < #outgoing_wrapper >; + type Error = #error_ty; + + async fn try_recv(&mut self) -> ::std::result::Result>, ()> { + match #support_crate ::poll!(self.recv()) { + #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), + #support_crate ::Poll::Pending => Ok(None), + } + } + + async fn recv(&mut self) -> ::std::result::Result, #error_ty> { + loop { + // If we have a message pending an overseer signal, we only poll for signals + // in the meantime. + if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { + if needs_signals_received <= self.signals_received.load() { + return Ok( #support_crate ::FromOverseer::Communication { msg }); + } else { + self.pending_incoming = Some((needs_signals_received, msg)); + + // wait for next signal. + let signal = self.signals.next().await + .ok_or(#support_crate ::OverseerError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; + + self.signals_received.inc(); + return Ok( #support_crate ::FromOverseer::Signal(signal)) + } + } + + let mut await_message = self.messages.next().fuse(); + let mut await_signal = self.signals.next().fuse(); + let signals_received = self.signals_received.load(); + let pending_incoming = &mut self.pending_incoming; + + // Otherwise, wait for the next signal or incoming message. + let from_overseer = #support_crate ::futures::select_biased! { + signal = await_signal => { + let signal = signal + .ok_or( #support_crate ::OverseerError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; + + #support_crate ::FromOverseer::Signal(signal) + } + msg = await_message => { + let packet = msg + .ok_or( #support_crate ::OverseerError::Context( + "Message channel is terminated and empty." + .to_owned() + ))?; + + if packet.signals_received > signals_received { + // wait until we've received enough signals to return this message. + *pending_incoming = Some((packet.signals_received, packet.message)); + continue; + } else { + // we know enough to return this message. + #support_crate ::FromOverseer::Communication { msg: packet.message} + } + } + }; + + if let #support_crate ::FromOverseer::Signal(_) = from_overseer { + self.signals_received.inc(); + } + + return Ok(from_overseer); + } + } + + fn sender(&mut self) -> &mut Self::Sender { + &mut self.to_subsystems + } + + fn spawn(&mut self, name: &'static str, s: Pin + Send>>) + -> ::std::result::Result<(), #error_ty> + { + self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { + name, + subsystem: Some(self.name()), + s, + }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; + Ok(()) + } + + fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + -> ::std::result::Result<(), #error_ty> + { + self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { + name, + subsystem: Some(self.name()), + s, + }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; + Ok(()) + } + } + } +} + pub(crate) fn impl_per_subsystem_helper_traits( info: &OverseerInfo, subsystem_ctx_name: &Ident, @@ -381,7 +529,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( outgoing_wrapper: &Ident, ) -> TokenStream { let all_messages_wrapper = &info.message_wrapper; - let signal = &info.extern_signal_ty; + let signal_ty = &info.extern_signal_ty; let error_ty = &info.extern_error_ty; let support_crate = info.support_crate_name(); @@ -424,7 +572,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( /// Accumuative trait for a particular subsystem wrapper. pub trait #subsystem_ctx_trait : SubsystemContext < Message = #consumes, - Signal = #signal, + Signal = #signal_ty, OutgoingMessages = #outgoing_wrapper, // Sender, Error = #error_ty, @@ -443,7 +591,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( where T: SubsystemContext < Message = #consumes, - Signal = #signal, + Signal = #signal_ty, OutgoingMessages = #outgoing_wrapper, // Sender Error = #error_ty, @@ -457,141 +605,19 @@ pub(crate) fn impl_per_subsystem_helper_traits( } }); - // impl the subsystem context trait - let impl_subsystem_ctx_trait = |consumes: Type, - outgoing: &[Type], - outgoing_wrapper: Type| - -> TokenStream { - let where_clause = quote! { - #consumes: AssociateOutgoing + ::std::fmt::Debug + Send + 'static, - #all_messages_wrapper: From< #outgoing_wrapper >, - #all_messages_wrapper: From< #consumes >, - #outgoing_wrapper: #( From< #outgoing > )+*, - }; - - quote! { - #[#support_crate ::async_trait] - impl #support_crate ::SubsystemContext for #subsystem_ctx_name < #consumes > - where - #where_clause - { - type Message = #consumes; - type Signal = #signal; - type OutgoingMessages = #outgoing_wrapper; - type Sender = #subsystem_sender_name < #outgoing_wrapper >; - type Error = #error_ty; - - async fn try_recv(&mut self) -> ::std::result::Result>, ()> { - match #support_crate ::poll!(self.recv()) { - #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), - #support_crate ::Poll::Pending => Ok(None), - } - } - - async fn recv(&mut self) -> ::std::result::Result, #error_ty> { - loop { - // If we have a message pending an overseer signal, we only poll for signals - // in the meantime. - if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { - if needs_signals_received <= self.signals_received.load() { - return Ok( #support_crate ::FromOverseer::Communication { msg }); - } else { - self.pending_incoming = Some((needs_signals_received, msg)); - - // wait for next signal. - let signal = self.signals.next().await - .ok_or(#support_crate ::OverseerError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; - - self.signals_received.inc(); - return Ok( #support_crate ::FromOverseer::Signal(signal)) - } - } - - let mut await_message = self.messages.next().fuse(); - let mut await_signal = self.signals.next().fuse(); - let signals_received = self.signals_received.load(); - let pending_incoming = &mut self.pending_incoming; - - // Otherwise, wait for the next signal or incoming message. - let from_overseer = #support_crate ::futures::select_biased! { - signal = await_signal => { - let signal = signal - .ok_or( #support_crate ::OverseerError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; - - #support_crate ::FromOverseer::Signal(signal) - } - msg = await_message => { - let packet = msg - .ok_or( #support_crate ::OverseerError::Context( - "Message channel is terminated and empty." - .to_owned() - ))?; - - if packet.signals_received > signals_received { - // wait until we've received enough signals to return this message. - *pending_incoming = Some((packet.signals_received, packet.message)); - continue; - } else { - // we know enough to return this message. - #support_crate ::FromOverseer::Communication { msg: packet.message} - } - } - }; - - if let #support_crate ::FromOverseer::Signal(_) = from_overseer { - self.signals_received.inc(); - } - - return Ok(from_overseer); - } - } - - fn sender(&mut self) -> &mut Self::Sender { - &mut self.to_subsystems - } - - fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - -> ::std::result::Result<(), #error_ty> - { - self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { - name, - subsystem: Some(self.name()), - s, - }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; - Ok(()) - } - - fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - -> ::std::result::Result<(), #error_ty> - { - self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { - name, - subsystem: Some(self.name()), - s, - }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; - Ok(()) - } - } - } - }; - - ts.extend(impl_subsystem_ctx_trait( + ts.extend(impl_subsystem_context_trait_for( parse_quote! { #consumes }, &Vec::from_iter(outgoing.iter().map(|path| { parse_quote! { #path } })), parse_quote! { #outgoing_wrapper }, + all_messages_wrapper, + subsystem_ctx_name, + subsystem_sender_name, + support_crate, + signal_ty, + error_ty, )); - - let empty_tuple: Type = parse_quote! { () }; - ts.extend(impl_subsystem_ctx_trait(empty_tuple.clone(), &[], empty_tuple)); - ts } @@ -602,7 +628,7 @@ pub(crate) fn impl_subsystem_context( subsystem_sender_name: &Ident, subsystem_ctx_name: &Ident, ) -> TokenStream { - let signal = &info.extern_signal_ty; + let signal_ty = &info.extern_signal_ty; let support_crate = info.support_crate_name(); let ts = quote! { @@ -616,7 +642,7 @@ pub(crate) fn impl_subsystem_context( #[derive(Debug)] #[allow(missing_docs)] pub struct #subsystem_ctx_name { - signals: #support_crate ::metered::MeteredReceiver< #signal >, + signals: #support_crate ::metered::MeteredReceiver< #signal_ty >, messages: SubsystemIncomingMessages< M >, to_subsystems: #subsystem_sender_name < ::OutgoingMessages >, to_overseer: #support_crate ::metered::UnboundedMeteredSender< @@ -633,7 +659,7 @@ pub(crate) fn impl_subsystem_context( { /// Create a new context. fn new( - signals: #support_crate ::metered::MeteredReceiver< #signal >, + signals: #support_crate ::metered::MeteredReceiver< #signal_ty >, messages: SubsystemIncomingMessages< M >, to_subsystems: ChannelsOut, to_overseer: #support_crate ::metered::UnboundedMeteredSender<#support_crate:: ToOverseer>, From 10427947eedb44e6d66bf7a72d8258d8e6a60b7d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 11:34:22 +0200 Subject: [PATCH 107/139] subsystem util --- node/subsystem-test-helpers/src/lib.rs | 11 +++++++---- node/subsystem-util/src/tests.rs | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index 624b0af73d55..d1729665c79e 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -318,10 +318,13 @@ pub struct ForwardSubsystem(pub mpsc::Sender); impl overseer::Subsystem for ForwardSubsystem where - M: std::fmt::Debug + Send + 'static, - Context: - overseer::SubsystemContext, - ::OutgoingMessages: From + Send, + M: overseer::AssociateOutgoing + std::fmt::Debug + Send + 'static, + Context: overseer::SubsystemContext< + Message = M, + Signal = OverseerSignal, + Error = SubsystemError, + OutgoingMessages = ::OutgoingMessages, + >, { fn start(mut self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 180cb384e2ca..12c9e21220d2 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -60,10 +60,10 @@ enum Error { impl JobTrait for FakeCollatorProtocolJob where - Sender: overseer::CollatorProtocolSender, + Sender: overseer::CollatorProtocolSenderTrait + std::marker::Unpin, { - type Consumes = CollatorProtocolMessage; - type OutgoingMessages = CollatorProtocolOutgoingMessages; + type ToJob = CollatorProtocolMessage; + type OutgoingMessages = overseer::CollatorProtocolOutgoingMessages; type Sender = Sender; type Error = Error; type RunArgs = bool; @@ -102,7 +102,10 @@ where } } -impl FakeCollatorProtocolJob { +impl FakeCollatorProtocolJob +where + Sender: overseer::CollatorProtocolSenderTrait, +{ async fn run_loop(mut self) -> Result<(), Error> { loop { match self.receiver.next().await { From 151c8e79099c4ac28a687492b0053ecc3d03842b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 13:30:44 +0200 Subject: [PATCH 108/139] fix mut --- node/collation-generation/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/collation-generation/src/tests.rs b/node/collation-generation/src/tests.rs index 563e3cfdce02..ca75666c85e3 100644 --- a/node/collation-generation/src/tests.rs +++ b/node/collation-generation/src/tests.rs @@ -296,7 +296,7 @@ mod handle_new_activations { *subsystem_sent_messages.lock().await = rx.collect().await; }); - let sent_messages = Arc::try_unwrap(sent_messages) + let mut sent_messages = Arc::try_unwrap(sent_messages) .expect("subsystem should have shut down by now") .into_inner(); From a8a89d715b4505125576c5af6cbaa38f86f2734f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 13:32:25 +0200 Subject: [PATCH 109/139] approval voting test fixes --- node/core/approval-voting/src/import.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index 0550ce5ed223..202e44464f0d 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -611,7 +611,7 @@ pub(crate) mod tests { use assert_matches::assert_matches; use merlin::Transcript; use polkadot_node_primitives::approval::{VRFOutput, VRFProof}; - use polkadot_node_subsystem::messages::AllMessages; + use polkadot_node_subsystem::messages::{AllMessages, ApprovalVotingMessage}; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::v2::{Id as ParaId, SessionInfo, ValidatorIndex}; @@ -726,7 +726,8 @@ pub(crate) mod tests { #[test] fn imported_block_info_is_good() { let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); + let (mut ctx, mut handle) = + make_subsystem_context::(pool.clone()); let session = 5; let session_info = dummy_session_info(session); @@ -849,7 +850,8 @@ pub(crate) mod tests { #[test] fn imported_block_info_fails_if_no_babe_vrf() { let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); + let (mut ctx, mut handle) = + make_subsystem_context::(pool.clone()); let session = 5; let session_info = dummy_session_info(session); @@ -952,7 +954,8 @@ pub(crate) mod tests { #[test] fn imported_block_info_fails_if_ancient_session() { let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); + let (mut ctx, mut handle) = + make_subsystem_context::(pool.clone()); let session = 5; @@ -1029,7 +1032,7 @@ pub(crate) mod tests { #[test] fn imported_block_info_extracts_force_approve() { let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); + let (mut ctx, mut handle) = make_subsystem_context(pool.clone()); let session = 5; let session_info = dummy_session_info(session); @@ -1160,7 +1163,8 @@ pub(crate) mod tests { let mut overlay_db = OverlayedBackend::new(&db); let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); + let (mut ctx, mut handle) = + make_subsystem_context::(pool.clone()); let session = 5; let irrelevant = 666; From 4d80c534f7f63383af3c0eb51e38d5506d4e23cf Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 13:32:36 +0200 Subject: [PATCH 110/139] fix overseer AllMessages gets an associateoutgoing type, useful for testing --- node/overseer/overseer-gen/examples/solo.rs | 1 - .../proc-macro/src/impl_subsystem_ctx_sender.rs | 4 ++++ node/subsystem-test-helpers/src/lib.rs | 12 ++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index adb6869a63ee..29dc2cd46233 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -2,7 +2,6 @@ //! A minimal demo to be used with cargo expand. -use overseer::SubsystemSender as _; use polkadot_overseer_gen::{self as overseer, SpawnNamed, *}; mod misc; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index 3b428c1c2858..2243a406b27f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -368,6 +368,10 @@ pub(crate) fn impl_associate_outgoing_messages_trait(all_messages_wrapper: &Iden impl AssociateOutgoing for () { type OutgoingMessages = (); } + + impl AssociateOutgoing for #all_messages_wrapper { + type OutgoingMessages = #all_messages_wrapper ; + } } } diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index d1729665c79e..b36c180a57b7 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -150,25 +150,25 @@ pub fn sender_receiver() -> (TestSubsystemSender, mpsc::UnboundedReceiver overseer::SubsystemSender for TestSubsystemSender +impl overseer::SubsystemSender for TestSubsystemSender where - AllMessages: From, - T: Send + 'static, + AllMessages: From, + OutgoingMessage: Send + 'static, { - async fn send_message(&mut self, msg: T) { + async fn send_message(&mut self, msg: OutgoingMessage) { self.tx.send(msg.into()).await.expect("test overseer no longer live"); } async fn send_messages(&mut self, msgs: I) where - I: IntoIterator + Send, + I: IntoIterator + Send, I::IntoIter: Send, { let mut iter = stream::iter(msgs.into_iter().map(|msg| Ok(msg.into()))); self.tx.send_all(&mut iter).await.expect("test overseer no longer live"); } - fn send_unbounded_message(&mut self, msg: T) { + fn send_unbounded_message(&mut self, msg: OutgoingMessage) { self.tx.unbounded_send(msg.into()).expect("test overseer no longer live"); } } From 3e70feb8e737f370ad78e51649bbedb204137161 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 13:33:10 +0200 Subject: [PATCH 111/139] fix rolling session window and other utils --- .../src/rolling_session_window.rs | 10 ++++---- node/subsystem-util/src/tests.rs | 23 +++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index d8a908a6901c..65921ba47b73 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -329,11 +329,11 @@ mod tests { let test_fut = { Box::pin(async move { let window = match window { - None => RollingSessionWindow::new(&mut sender, TEST_WINDOW_SIZE, hash) + None => RollingSessionWindow::new(sender.clone(), TEST_WINDOW_SIZE, hash) .await .unwrap(), Some(mut window) => { - window.cache_session_info_for_head(&mut sender, hash).await.unwrap(); + window.cache_session_info_for_head(sender, hash).await.unwrap(); window }, }; @@ -503,8 +503,9 @@ mod tests { let hash = header.hash(); let test_fut = { + let sender = ctx.sender().clone(); Box::pin(async move { - let res = RollingSessionWindow::new(ctx.sender(), TEST_WINDOW_SIZE, hash).await; + let res = RollingSessionWindow::new(sender, TEST_WINDOW_SIZE, hash).await; assert!(res.is_err()); }) }; @@ -563,8 +564,9 @@ mod tests { let test_fut = { Box::pin(async move { + let sender = ctx.sender().clone(); let window = - RollingSessionWindow::new(ctx.sender(), TEST_WINDOW_SIZE, hash).await.unwrap(); + RollingSessionWindow::new(sender, TEST_WINDOW_SIZE, hash).await.unwrap(); assert_eq!(window.earliest_session, session); assert_eq!(window.session_info, vec![dummy_session_info(session)]); diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 12c9e21220d2..5d25af8ef097 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -60,7 +60,12 @@ enum Error { impl JobTrait for FakeCollatorProtocolJob where - Sender: overseer::CollatorProtocolSenderTrait + std::marker::Unpin, + Sender: overseer::CollatorProtocolSenderTrait + + std::marker::Unpin + + overseer::SubsystemSender, + JobSender: overseer::CollatorProtocolSenderTrait + + std::marker::Unpin + + overseer::SubsystemSender, { type ToJob = CollatorProtocolMessage; type OutgoingMessages = overseer::CollatorProtocolOutgoingMessages; @@ -86,12 +91,16 @@ where FakeCollatorProtocolJob { receiver, _phantom: std::marker::PhantomData:: }; if run_args { - sender - .send_message(CollatorProtocolMessage::Invalid( - dummy_hash(), - dummy_candidate_receipt(dummy_hash()), - )) - .await; + // FIXME XXX + // this was a hack for testing, for sending a message to a subsystem itself + // on a bounded channel which is outragiously stupid, even for a test case + // sender + // .send_message(CollatorProtocolMessage::Invalid( + // dummy_hash(), + // dummy_candidate_receipt(dummy_hash()), + // )) + // .await; + unimplemented!("Probably delete this testcase entirely TODO") } // it isn't necessary to break run_loop into its own function, From 714b045defc868679effabff44d570c67112c111 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 13:47:54 +0200 Subject: [PATCH 112/139] undo change for util test --- node/subsystem-util/src/tests.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 5d25af8ef097..2a1a95a88458 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +#![cfg(test)] + use super::*; use assert_matches::assert_matches; use executor::block_on; @@ -91,16 +93,12 @@ where FakeCollatorProtocolJob { receiver, _phantom: std::marker::PhantomData:: }; if run_args { - // FIXME XXX - // this was a hack for testing, for sending a message to a subsystem itself - // on a bounded channel which is outragiously stupid, even for a test case - // sender - // .send_message(CollatorProtocolMessage::Invalid( - // dummy_hash(), - // dummy_candidate_receipt(dummy_hash()), - // )) - // .await; - unimplemented!("Probably delete this testcase entirely TODO") + sender + .send_message(CollatorProtocolMessage::Invalid( + dummy_hash(), + dummy_candidate_receipt(dummy_hash()), + )) + .await; } // it isn't necessary to break run_loop into its own function, From 19a826373142d48949bf128808616410743131b5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 13:48:58 +0200 Subject: [PATCH 113/139] foo fmt --- node/subsystem-util/src/rolling_session_window.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 65921ba47b73..c7793e7efcca 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -279,7 +279,10 @@ async fn load_all_sessions( mod tests { use super::*; use assert_matches::assert_matches; - use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; + use polkadot_node_subsystem::{ + messages::{AllMessages, AvailabilityRecoveryMessage}, + SubsystemContext, + }; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_primitives::v2::Header; use sp_core::testing::TaskExecutor; From cfdfcff56724dab4c20e524f7da5089297972c75 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 14:26:25 +0200 Subject: [PATCH 114/139] fix network bridge test cases --- node/network/bridge/src/lib.rs | 15 +++++++++------ node/network/bridge/src/tests.rs | 5 +++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index b3848f1b1448..65a828cbb300 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -1062,19 +1062,22 @@ fn dispatch_validation_event_to_all_unbounded( event .focus() .ok() - .map(ApprovalDistributionMessage::from) + .map(StatementDistributionMessage::from) .and_then(|msg| Some(sender.send_unbounded_message(msg))); - event .focus() .ok() .map(BitfieldDistributionMessage::from) .and_then(|msg| Some(sender.send_unbounded_message(msg))); - event .focus() .ok() - .map(StatementDistributionMessage::from) + .map(ApprovalDistributionMessage::from) + .and_then(|msg| Some(sender.send_unbounded_message(msg))); + event + .focus() + .ok() + .map(GossipSupportMessage::from) .and_then(|msg| Some(sender.send_unbounded_message(msg))); } @@ -1095,12 +1098,12 @@ async fn dispatch_validation_events_to_all( I::IntoIter: Send, { for event in events { - sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; - sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; sender .send_messages(event.focus().map(StatementDistributionMessage::from)) .await; + sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; + sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; } } diff --git a/node/network/bridge/src/tests.rs b/node/network/bridge/src/tests.rs index f9752900bbb5..62291be8c662 100644 --- a/node/network/bridge/src/tests.rs +++ b/node/network/bridge/src/tests.rs @@ -313,8 +313,9 @@ async fn assert_sends_validation_event_to_all( event: NetworkBridgeEvent, virtual_overseer: &mut TestSubsystemContextHandle, ) { - // Ordering must match the enum variant order - // in `AllMessages`. + // Ordering must be consistent across: + // `fn dispatch_validation_event_to_all_unbounded` + // `dispatch_validation_events_to_all` assert_matches!( virtual_overseer.recv().await, AllMessages::StatementDistribution( From 286db5ee39b968e5599fc85d09ae4930512fb18a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 14:34:32 +0200 Subject: [PATCH 115/139] chore: spellcheck --- .../overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs | 2 +- node/overseer/overseer-gen/proc-macro/src/subsystem.rs | 4 ++-- node/subsystem-util/src/lib.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index 2243a406b27f..d07b0b0761d0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -573,7 +573,7 @@ pub(crate) fn impl_per_subsystem_helper_traits( }; ts.extend(quote! { - /// Accumuative trait for a particular subsystem wrapper. + /// Accumulative trait for a particular subsystem wrapper. pub trait #subsystem_ctx_trait : SubsystemContext < Message = #consumes, Signal = #signal_ty, diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 0a844dc2eb99..efc1cab7cfa7 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -40,7 +40,7 @@ //! where `support_crate` is either equivalent to `somewhere` or derived from the cargo manifest. //! //! -//! ## Add aditional trait bounds for a generic `Context` via `contextbounds` +//! ## Add additional trait bounds for a generic `Context` via `contextbounds` //! //! ### To an `ImplItem` //! @@ -93,7 +93,7 @@ use super::{parse::*, *}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum MakeSubsystem { - /// Impl `trait Subsystem` and apply the trait bounds to the `Context` generic. + /// Implements `trait Subsystem` and apply the trait bounds to the `Context` generic. /// /// Relevant to `impl Item` only. ImplSubsystemTrait, diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 0648ac9bcdb3..dcc7c5615431 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -484,7 +484,7 @@ pub trait JobTrait: Unpin + Sized { /// Message type used to send messages to the job. type ToJob: 'static + BoundToRelayParent + Send; - /// The set of outgoing messages to be accumalted into. + /// The set of outgoing messages to be accumulated into. type OutgoingMessages: 'static + Send; /// The sender to send outgoing messages. From dbcc8b9778ee5653462a341eafc408807bbdcad2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 9 May 2022 15:06:45 +0200 Subject: [PATCH 116/139] fix docs --- .../src/impl_subsystem_ctx_sender.rs | 29 +++++++++++++++---- .../overseer-gen/proc-macro/src/overseer.rs | 2 +- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index d07b0b0761d0..fc0e056f84ca 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -51,7 +51,8 @@ fn graphviz( Ok(()) } -pub(crate) fn impl_subsystem(info: &OverseerInfo) -> Result { +/// Generates all subsystem types and related accumulation traits. +pub(crate) fn impl_subsystem_types_all(info: &OverseerInfo) -> Result { let mut ts = TokenStream::new(); let overseer_name = &info.overseer_name; @@ -262,6 +263,8 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu Ok(ts) } +/// Create the clonable subsystem sender type and implements `trait SubsystemSender` +/// for the `#outgoing_wrappers: From` with the proper associted types. pub(crate) fn impl_subsystem_sender( support_crate: &Path, outgoing_wrappers: impl IntoIterator, @@ -353,6 +356,7 @@ pub(crate) fn impl_subsystem_sender( ts } +/// Define the `trait AssociateOutgoing` and implement it for `#all_messages_wrapper` and `()`. pub(crate) fn impl_associate_outgoing_messages_trait(all_messages_wrapper: &Ident) -> TokenStream { quote! { /// Binds a generated type covering all declared outgoing messages, @@ -365,19 +369,27 @@ pub(crate) fn impl_associate_outgoing_messages_trait(all_messages_wrapper: &Iden type OutgoingMessages: Into< #all_messages_wrapper > + ::std::fmt::Debug + Send; } + // Helper for tests, where nothing is ever sent. impl AssociateOutgoing for () { type OutgoingMessages = (); } + // Helper for tests, allows sending of arbitrary messages give + // an test context. impl AssociateOutgoing for #all_messages_wrapper { type OutgoingMessages = #all_messages_wrapper ; } } } -/// A workaround until default associated types actually work. +/// Implement `AssociateOutgoing` for `#consumes` being handled by a particular subsystem. +/// +/// Binds the outgoing messages to the inbound message. +/// +/// Note: Works, since there is a 1:1 relation between inbound message type and subsystem declarations. +/// Note: A workaround until default associated types work in `rustc`. pub(crate) fn impl_associate_outgoing_messages( - message: &Path, + consumes: &Path, outgoing_wrapper: &Ident, ) -> TokenStream { quote! { @@ -385,12 +397,14 @@ pub(crate) fn impl_associate_outgoing_messages( type OutgoingMessages = #outgoing_wrapper; } - impl AssociateOutgoing for #message { + impl AssociateOutgoing for #consumes { type OutgoingMessages = #outgoing_wrapper; } } } +/// Implement `trait SubsystemContext` for a particular subsystem context, +/// that is generated by the proc macro too. pub(crate) fn impl_subsystem_context_trait_for( consumes: Type, outgoing: &[Type], @@ -522,6 +536,8 @@ pub(crate) fn impl_subsystem_context_trait_for( } } +/// Implement the additional subsystem accumulation traits, for simplified usage, +/// i.e. `${Subsystem}SenderTrait` and `${Subsystem}ContextTrait`. pub(crate) fn impl_per_subsystem_helper_traits( info: &OverseerInfo, subsystem_ctx_name: &Ident, @@ -625,8 +641,9 @@ pub(crate) fn impl_per_subsystem_helper_traits( ts } -/// Implement a builder pattern for the `Overseer`-type, -/// which acts as the gateway to constructing the overseer. +/// Generate the subsystem context type and provide `fn new` on it. +/// +/// Note: The generated `fn new` is used by the [builder pattern](../impl_builder.rs). pub(crate) fn impl_subsystem_context( info: &OverseerInfo, subsystem_sender_name: &Ident, diff --git a/node/overseer/overseer-gen/proc-macro/src/overseer.rs b/node/overseer/overseer-gen/proc-macro/src/overseer.rs index 54944053bbb6..eff28a38c29a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/overseer.rs @@ -48,7 +48,7 @@ pub(crate) fn impl_overseer_gen( additive.extend(impl_overseen_subsystem(&info)); additive.extend(impl_channels_out_struct(&info)); - additive.extend(impl_subsystem(&info)?); + additive.extend(impl_subsystem_types_all(&info)?); additive.extend(impl_message_wrapper_enum(&info)?); From d6754b5704f75b7e22f575af1b74adeca11a2625 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 10 May 2022 16:56:18 +0200 Subject: [PATCH 117/139] rebase fallout --- node/core/provisioner/src/lib.rs | 43 ------------------- node/core/provisioner/src/onchain_disputes.rs | 15 ++++--- 2 files changed, 9 insertions(+), 49 deletions(-) diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 8bc5491b0216..afbdf458746e 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -95,49 +95,6 @@ impl InherentAfter { } } -/// Errors in the provisioner. -#[derive(Debug, Error)] -#[allow(missing_docs)] -pub enum Error { - #[error(transparent)] - Util(#[from] util::Error), - - #[error("failed to get availability cores")] - CanceledAvailabilityCores(#[source] oneshot::Canceled), - - #[error("failed to get persisted validation data")] - CanceledPersistedValidationData(#[source] oneshot::Canceled), - - #[error("failed to get block number")] - CanceledBlockNumber(#[source] oneshot::Canceled), - - #[error("failed to get backed candidates")] - CanceledBackedCandidates(#[source] oneshot::Canceled), - - #[error("failed to get votes on dispute")] - CanceledCandidateVotes(#[source] oneshot::Canceled), - - #[error(transparent)] - ChainApi(#[from] ChainApiError), - - #[error(transparent)] - Runtime(#[from] RuntimeApiError), - - #[error("failed to send message to ChainAPI")] - ChainApiMessageSend(#[source] mpsc::SendError), - - #[error("failed to send message to CandidateBacking to get backed candidates")] - GetBackedCandidatesSend(#[source] mpsc::SendError), - - #[error("failed to send return message with Inherents")] - InherentDataReturnChannel, - - #[error( - "backed candidate does not correspond to selected candidate; check logic in provisioner" - )] - BackedCandidateOrderingProblem, -} - /// Provisioner run arguments. #[derive(Debug, Clone, Copy)] pub struct ProvisionerConfig; diff --git a/node/core/provisioner/src/onchain_disputes.rs b/node/core/provisioner/src/onchain_disputes.rs index e5b736b600b3..6c83d77fc9c9 100644 --- a/node/core/provisioner/src/onchain_disputes.rs +++ b/node/core/provisioner/src/onchain_disputes.rs @@ -15,14 +15,17 @@ // along with Polkadot. If not, see . use crate::error::GetOnchainDisputesError; -use polkadot_node_subsystem::SubsystemSender; +use polkadot_node_subsystem::overseer; use polkadot_primitives::v2::{CandidateHash, DisputeState, Hash, SessionIndex}; use std::collections::HashMap; -pub async fn get_onchain_disputes( - _sender: &mut impl SubsystemSender, +pub async fn get_onchain_disputes( + _sender: &mut Sender, _relay_parent: Hash, -) -> Result, GetOnchainDisputesError> { +) -> Result, GetOnchainDisputesError> +where + Sender: overseer::ProvisionerSenderTrait, +{ let _onchain = Result::< HashMap<(SessionIndex, CandidateHash), DisputeState>, GetOnchainDisputesError, @@ -46,8 +49,8 @@ mod staging_impl { }; /// Gets the on-chain disputes at a given block number and returns them as a `HashSet` so that searching in them is cheap. - pub async fn get_onchain_disputes( - sender: &mut impl SubsystemSender, + pub async fn get_onchain_disputes( + sender: &mut Sender, relay_parent: Hash, ) -> Result, GetOnchainDisputesError> { gum::trace!(target: LOG_TARGET, ?relay_parent, "Fetching on-chain disputes"); From 727dfcc29492751a57d2e533e829f6596cce9098 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 10 May 2022 17:42:26 +0200 Subject: [PATCH 118/139] fixins --- node/overseer/examples/minimal-example.rs | 13 +++--- node/overseer/overseer-gen/examples/duo.rs | 41 +++++++++++++------ node/overseer/overseer-gen/examples/misc.rs | 10 +++-- node/overseer/overseer-gen/examples/solo.rs | 26 +++++++++--- .../overseer-gen/proc-macro/src/overseer.rs | 2 +- .../src/rolling_session_window.rs | 2 +- 6 files changed, 63 insertions(+), 31 deletions(-) diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 036594306d63..c3cbbd3b289a 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -24,14 +24,12 @@ use std::time::Duration; use ::test_helpers::{dummy_candidate_descriptor, dummy_hash}; use polkadot_node_primitives::{BlockData, PoV}; -use polkadot_node_subsystem_types::messages::{ - CandidateBackingMessage, CandidateValidationMessage, -}; +use polkadot_node_subsystem_types::messages::CandidateValidationMessage; use polkadot_overseer::{ self as overseer, dummy::dummy_overseer_builder, gen::{FromOverseer, SpawnedSubsystem}, - HeadSupportsParachains, OverseerSignal, SubsystemError, + HeadSupportsParachains, SubsystemError, }; use polkadot_primitives::v2::{CandidateReceipt, Hash}; @@ -84,9 +82,9 @@ impl Subsystem1 { } } -#[overseer::contextbounds(CandidateBacking, error = SubsystemError, prefix = self::overseer)] -impl Subsystem1 { - fn start(self, ctx: Context) -> SpawnedSubsystem { +#[overseer::subsystem(CandidateBacking, error = SubsystemError, prefix = self::overseer)] +impl Subsystem1 { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { Self::run(ctx).await; Ok(()) @@ -156,6 +154,7 @@ fn main() { .unwrap() .replace_candidate_validation(|_| Subsystem2) .replace_candidate_backing(|orig| orig) + .replace_candidate_backing(|_orig| Subsystem1) .build() .unwrap(); diff --git a/node/overseer/overseer-gen/examples/duo.rs b/node/overseer/overseer-gen/examples/duo.rs index 66c33ddcd0a5..b63a7df603e1 100644 --- a/node/overseer/overseer-gen/examples/duo.rs +++ b/node/overseer/overseer-gen/examples/duo.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] // overseer events are not used + //! A dummy to be used with cargo expand use polkadot_overseer_gen::{self as overseer, SpawnNamed, *}; @@ -57,16 +59,31 @@ struct Duo { } fn main() { - let (overseer, _handle): (Duo<_, f64>, _) = Duo::builder() - .sub0(AwesomeSubSys::default()) - .plinkos(Fortified::default()) - .i_like_pi(::std::f64::consts::PI) - .i_like_generic(42.0) - .i_like_hash(HashMap::new()) - .spawner(DummySpawner) - .build() - .unwrap(); - assert_eq!(overseer.i_like_pi.floor() as i8, 3); - assert_eq!(overseer.i_like_generic.floor() as i8, 42); - assert_eq!(overseer.i_like_hash.len() as i8, 0); + use futures::{executor, pin_mut}; + + executor::block_on(async move { + let (overseer, _handle): (Duo<_, f64>, _) = Duo::builder() + .sub0(AwesomeSubSys::default()) + .plinkos(Fortified::default()) + .i_like_pi(::std::f64::consts::PI) + .i_like_generic(42.0) + .i_like_hash(HashMap::new()) + .spawner(DummySpawner) + .build() + .unwrap(); + + assert_eq!(overseer.i_like_pi.floor() as i8, 3); + assert_eq!(overseer.i_like_generic.floor() as i8, 42); + assert_eq!(overseer.i_like_hash.len() as i8, 0); + + let overseer_fut = overseer + .running_subsystems + .into_future() + .timeout(std::time::Duration::from_millis(300)) + .fuse(); + + pin_mut!(overseer_fut); + + overseer_fut.await + }); } diff --git a/node/overseer/overseer-gen/examples/misc.rs b/node/overseer/overseer-gen/examples/misc.rs index cfbb56d04005..87f674e14134 100644 --- a/node/overseer/overseer-gen/examples/misc.rs +++ b/node/overseer/overseer-gen/examples/misc.rs @@ -1,5 +1,11 @@ use polkadot_overseer_gen::{SpawnNamed, *}; +#[derive(Debug, Clone, Copy)] +pub enum SigSigSig { + Conclude, + Foo, +} + #[derive(Debug, Clone)] pub struct DummySpawner; @@ -23,10 +29,6 @@ impl SpawnNamed for DummySpawner { } } -/// A signal sent by the overseer. -#[derive(Debug, Clone)] -pub struct SigSigSig; - /// The external event. #[derive(Debug, Clone)] pub struct EvX; diff --git a/node/overseer/overseer-gen/examples/solo.rs b/node/overseer/overseer-gen/examples/solo.rs index 29dc2cd46233..74ecdc9d16ba 100644 --- a/node/overseer/overseer-gen/examples/solo.rs +++ b/node/overseer/overseer-gen/examples/solo.rs @@ -1,4 +1,4 @@ -#![allow(unused_assignments)] +#![allow(dead_code)] // overseer events are not used //! A minimal demo to be used with cargo expand. @@ -32,9 +32,23 @@ impl Fortified { } fn main() { - let (overseer, _handle): (Solo<_>, _) = Solo::builder() - .goblin_tower(Fortified::default()) - .spawner(DummySpawner) - .build() - .unwrap(); + use futures::{executor, pin_mut}; + + executor::block_on(async move { + let (overseer, _handle): (Solo<_>, _) = Solo::builder() + .goblin_tower(Fortified::default()) + .spawner(DummySpawner) + .build() + .unwrap(); + + let overseer_fut = overseer + .running_subsystems + .into_future() + .timeout(std::time::Duration::from_millis(300)) + .fuse(); + + pin_mut!(overseer_fut); + + overseer_fut.await + }); } diff --git a/node/overseer/overseer-gen/proc-macro/src/overseer.rs b/node/overseer/overseer-gen/proc-macro/src/overseer.rs index eff28a38c29a..7af4904c15e9 100644 --- a/node/overseer/overseer-gen/proc-macro/src/overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/overseer.rs @@ -56,7 +56,7 @@ pub(crate) fn impl_overseer_gen( .add_comment("Generated overseer code by `#[overlord(..)]`".to_owned()) .dry(!cfg!(feature = "expand")) .verbose(true) - .fmt(expander::Edition::_2021) + .fmt(expander::RustFmt::No) .write_to_out_dir(additive) .expect("Expander does not fail due to IO in OUT_DIR. qed"); diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index c7793e7efcca..ba8a62d08118 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -327,7 +327,7 @@ mod tests { let hash = header.hash(); - let mut sender = ctx.sender(); + let sender = ctx.sender(); let test_fut = { Box::pin(async move { From 1b939416b46dbe20a77c979edfcf0e91bb61c9ad Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 10 May 2022 17:42:56 +0200 Subject: [PATCH 119/139] cannot format generated files, since we still require nightly fmt Using stable fmt with out config causes dozends of errors --- node/overseer/overseer-gen/proc-macro/src/overseer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/overseer.rs b/node/overseer/overseer-gen/proc-macro/src/overseer.rs index 7af4904c15e9..2fef876ef5a3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/overseer.rs @@ -56,7 +56,7 @@ pub(crate) fn impl_overseer_gen( .add_comment("Generated overseer code by `#[overlord(..)]`".to_owned()) .dry(!cfg!(feature = "expand")) .verbose(true) - .fmt(expander::RustFmt::No) + // .fmt(expander::Edition::_2021) .write_to_out_dir(additive) .expect("Expander does not fail due to IO in OUT_DIR. qed"); From cf592396672e9b594af1c4af039fa6b609c1731e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 10 May 2022 18:09:50 +0200 Subject: [PATCH 120/139] add explaining comment --- node/overseer/overseer-gen/proc-macro/src/overseer.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node/overseer/overseer-gen/proc-macro/src/overseer.rs b/node/overseer/overseer-gen/proc-macro/src/overseer.rs index 2fef876ef5a3..99a74d0c476b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/overseer.rs @@ -56,6 +56,9 @@ pub(crate) fn impl_overseer_gen( .add_comment("Generated overseer code by `#[overlord(..)]`".to_owned()) .dry(!cfg!(feature = "expand")) .verbose(true) + // once all our needed format options are available on stable + // we should enabled this again, until then too many warnings + // are generated // .fmt(expander::Edition::_2021) .write_to_out_dir(additive) .expect("Expander does not fail due to IO in OUT_DIR. qed"); From eafd859461a9b1eaf5e3b8524d42758e85812f3d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 10 May 2022 18:15:22 +0200 Subject: [PATCH 121/139] please cargo spellcheck --- bridges/.config/lingua.dic | 2 ++ .../overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bridges/.config/lingua.dic b/bridges/.config/lingua.dic index f4b4d55d6e88..223a962006a2 100644 --- a/bridges/.config/lingua.dic +++ b/bridges/.config/lingua.dic @@ -16,6 +16,7 @@ Best/MS BlockId BlockNumber BridgeStorage +clonable CLI/MS Chain1 Chain2 @@ -177,6 +178,7 @@ plancks polkadot/MS pov-block/MS precommit +proc-macro/MS prometheus proxying provisioner/MS diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index fc0e056f84ca..26e9be8914f2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -264,7 +264,7 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu } /// Create the clonable subsystem sender type and implements `trait SubsystemSender` -/// for the `#outgoing_wrappers: From` with the proper associted types. +/// for the `#outgoing_wrappers: From` with the proper associated types. pub(crate) fn impl_subsystem_sender( support_crate: &Path, outgoing_wrappers: impl IntoIterator, @@ -404,7 +404,7 @@ pub(crate) fn impl_associate_outgoing_messages( } /// Implement `trait SubsystemContext` for a particular subsystem context, -/// that is generated by the proc macro too. +/// that is generated by the proc-macro too. pub(crate) fn impl_subsystem_context_trait_for( consumes: Type, outgoing: &[Type], From 9b32305a3342e120c74bb4c43168824e7b3acb85 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 10 May 2022 18:27:35 +0200 Subject: [PATCH 122/139] docs --- node/overseer/overseer-gen/README.md | 7 ++++--- node/overseer/overseer-gen/adr/01-adr.md | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 node/overseer/overseer-gen/adr/01-adr.md diff --git a/node/overseer/overseer-gen/README.md b/node/overseer/overseer-gen/README.md index ce03897f804c..21227049d731 100644 --- a/node/overseer/overseer-gen/README.md +++ b/node/overseer/overseer-gen/README.md @@ -11,17 +11,18 @@ declarative. ```rust #[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)] pub struct Overseer { - #[subsystem(MsgA)] + #[subsystem(MsgA, sends: [MsgB])] sub_a: AwesomeSubSysA, - #[subsystem(MsgB)] + #[subsystem(MsgB, sends: [MsgA])] sub_b: AwesomeSubSysB, } ``` * Each subsystem is annotated with `#[subsystem(_)]` where `MsgA` respectively `MsgB` are the messages being consumed by that particular subsystem. Each of those subsystems is required to implement the subsystem -trait. +trait with the correct trait bounds. Commonly this is achieved +by using `#[subsystem]` and `#[contextbounds]` macro. * `error=` tells the overseer to use the user provided error type, if not provided a builtin one is used. Note that this is the one error type used throughout all calls, so make sure it does impl `From` for all other error types `E` that are relevant to your application. * `event=` declares an external event type, that injects certain events diff --git a/node/overseer/overseer-gen/adr/01-adr.md b/node/overseer/overseer-gen/adr/01-adr.md new file mode 100644 index 000000000000..410f5a699b29 --- /dev/null +++ b/node/overseer/overseer-gen/adr/01-adr.md @@ -0,0 +1,21 @@ +# Limit outgoing messages + +## Status + +Accepted + implemented. + +## Context + +Previously, there was no way to limit and hence reason about a subset of subsystems, and if they form a cycle. Limiting the outgoing message types is a first step to create respective graphs and use classic graph algorithms to detect those and leave it to the user to resolve these. + +## Decision + +Annotate the `#[overlord]` inner `#[subsystem(..)]` annotation +with an aditional set of outgoing messages and enforce this via more fine grained trait bounds on the `Sender` and `::Sender` bounds. + +## Consequences + +* A graph will be spawn for every compilation under the `OUT_DIR` of the crate where `#[overlord]` is specified. +* Each subsystem has a consuming message which is often referred to as generic `M` (no change on that, is as before), but now we have trait `AssociateOutgoing { type OutgoingMessages = ..; }` which defines an outgoing helper `enum` that is generated with an ident constructed as `${Subsystem}OutgoingMessages` where `${Subsystem}` is the subsystem identifier as used in the overseer declaration. `${Subsystem}OutgoingMessages` is used throughout everywhere to constrain the outgoing messages (commonly referred to as `OutgoingMessage` generic bounded by `${Subsystem}OutgoingMessages: From` or `::OutgoingMessages: From`. It's what allows the construction of the graph and compile time verification. +* `${Subsystem}SenderTrait` and `${Subsystem}ContextTrait` are accumulation traits or wrapper traits, that combine over all annotated M or `OutgoingMessages` from the overseer declaration or their respective outgoing types. It is usage convenience and assures consistency within a subsystem while also maintaining a single source of truth for which messages can be sent by a particular subsystem. Note that this is sidestepped for the test subsystem, which may consume `gen=AllMessages`, the global message wrapper type. +* `Job`-based subsystems, being on their way out, are patched, but they now are generic over the `Sender` type, leaking that type. From 80b86e50890675f9fea13d3192b3b8ef6a660083 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 10:31:13 +0200 Subject: [PATCH 123/139] do not attempt to compile expansion examples which are not fully correct rust code --- .../overseer-gen/proc-macro/src/subsystem.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index efc1cab7cfa7..ff13cf4bc8e3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -19,14 +19,16 @@ //! //! ## Implement `trait Subsystem` via `subsystem` //! -//! ```rust +//! ```ignore //! #[subsystem(Foo, error = Yikes, prefix = somewhere)] //! impl BarSubsystem { //! .. //! } -//! ``` expands to +//! ``` +//! +//! expands to //! -//! ```rust +//! ```ignore //! impl support_crate::Subsystem for BarSubsystem //! where //! Context: somewhere::FooSubsystemTrait, @@ -44,7 +46,7 @@ //! //! ### To an `ImplItem` //! -//! ```rust +//! ```ignore //! #[contextbounds(Foo, prefix = somewhere)] //! impl X { //! .. @@ -53,7 +55,7 @@ //! //! expands to //! -//! ```rust +//! ```ignore //! impl X //! where //! Context: somewhere::FooSubsystemTrait, @@ -66,7 +68,7 @@ //! //! ### To a free standing `Fn` (not a method, that's covered by the above) //! -//! ```rust +//! ```ignore //! #[contextbounds(Foo, prefix = somewhere)] //! fn do_smth(context: &mut Context) { //! .. @@ -75,7 +77,7 @@ //! //! expands to //! -//! ```rust +//! ```ignore //! fn do_smth(context: &mut Context) //! where //! Context: somewhere::FooSubsystemTrait, From cfb77e94d4a731612e6b1ae0c3f9206096c857be Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 11:03:20 +0200 Subject: [PATCH 124/139] make doc tests work --- Cargo.lock | 3 + .../overseer-gen/proc-macro/Cargo.toml | 3 + .../src/parse/parse_overseer_attr.rs | 2 +- .../overseer-gen/proc-macro/src/subsystem.rs | 98 +++++++++++++++++-- node/overseer/overseer-gen/src/lib.rs | 1 + 5 files changed, 100 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b106534ac7f6..8f80291e9983 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7029,10 +7029,13 @@ dependencies = [ "assert_matches", "expander 0.0.6", "petgraph", + "polkadot-overseer-gen", "proc-macro-crate", "proc-macro2", "quote", "syn", + "thiserror", + "tracing-gum", ] [[package]] diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml index e1690807a7c9..7c6dcbd1b4ee 100644 --- a/node/overseer/overseer-gen/proc-macro/Cargo.toml +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -21,6 +21,9 @@ petgraph = "0.6.0" [dev-dependencies] assert_matches = "1.5.0" +polkadot-overseer-gen = { path = "../" } +thiserror = "1" +gum = { package = "tracing-gum", path = "../../../gum" } [features] default = ["graph", "expand"] diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs index 633a2cef4e24..e5cb30dd7513 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_attr.rs @@ -174,7 +174,7 @@ impl Parse for OverseerAttrArgs { let error = extract_variant!(unique, ExternErrorType; err = "Must declare the overseer error type via `error=..`.")?; let event = extract_variant!(unique, ExternEventType; err = "Must declare the overseer event type via `event=..`.")?; - let signal = extract_variant!(unique, ExternOverseerSignalType; err = "Must declare the overseer signal type via `span=..`.")?; + let signal = extract_variant!(unique, ExternOverseerSignalType; err = "Must declare the overseer signal type via `signal=..`.")?; let message_wrapper = extract_variant!(unique, MessageWrapperName; err = "Must declare the overseer generated wrapping message type via `gen=..`.")?; let outgoing = extract_variant!(unique, OutgoingType); diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index ff13cf4bc8e3..2324217f449d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -19,23 +19,109 @@ //! //! ## Implement `trait Subsystem` via `subsystem` //! -//! ```ignore +//! ```no_run +//! # use polkadot_overseer_gen_proc_macro::subsystem; +//! # mod somewhere { +//! # use polkadot_overseer_gen_proc_macro::overlord; +//! # pub use polkadot_overseer_gen::*; +//! # +//! # #[derive(Debug, thiserror::Error)] +//! # #[error("Yikes!")] +//! # pub struct Yikes; +//! # impl From for Yikes { +//! # fn from(_: OverseerError) -> Yikes { Yikes } +//! # } +//! # impl From for Yikes { +//! # fn from(_: mpsc::SendError) -> Yikes { Yikes } +//! # } +//! # +//! # #[derive(Debug)] +//! # pub struct Eve; +//! # +//! # #[derive(Debug, Clone)] +//! # pub struct Sig; +//! # +//! # #[derive(Debug, Clone, Copy)] +//! # pub struct A; +//! # #[derive(Debug, Clone, Copy)] +//! # pub struct B; +//! # +//! # #[overlord(signal=Sig, gen=AllOfThem, event=Eve, error=Yikes)] +//! # pub struct Wonderland { +//! # #[subsystem(A, sends: [B])] +//! # foo: Foo, +//! # #[subsystem(B, sends: [A])] +//! # bar: Bar, +//! # } +//! # } +//! # use somewhere::{Yikes, SpawnedSubsystem}; +//! # +//! # struct FooSubsystem; +//! # //! #[subsystem(Foo, error = Yikes, prefix = somewhere)] -//! impl BarSubsystem { -//! .. +//! impl FooSubsystem { +//! fn start(self, context: Context) -> SpawnedSubsystem { +//! // .. +//! # let _ = context; +//! # unimplemented!() +//! } //! } //! ``` //! //! expands to //! -//! ```ignore -//! impl support_crate::Subsystem for BarSubsystem +//! ```no_run +//! # use polkadot_overseer_gen_proc_macro::subsystem; +//! # mod somewhere { +//! # use polkadot_overseer_gen_proc_macro::overlord; +//! # pub use polkadot_overseer_gen::*; +//! # +//! # #[derive(Debug, thiserror::Error)] +//! # #[error("Yikes!")] +//! # pub struct Yikes; +//! # impl From for Yikes { +//! # fn from(_: OverseerError) -> Yikes { Yikes } +//! # } +//! # impl From for Yikes { +//! # fn from(_: mpsc::SendError) -> Yikes { Yikes } +//! # } +//! # +//! # #[derive(Debug)] +//! # pub struct Eve; +//! # +//! # #[derive(Debug, Clone)] +//! # pub struct Sig; +//! # +//! # #[derive(Debug, Clone, Copy)] +//! # pub struct A; +//! # #[derive(Debug, Clone, Copy)] +//! # pub struct B; +//! # +//! # #[overlord(signal=Sig, gen=AllOfThem, event=Eve, error=Yikes)] +//! # pub struct Wonderland { +//! # #[subsystem(A, sends: [B])] +//! # foo: Foo, +//! # #[subsystem(B, sends: [A])] +//! # bar: Bar, +//! # } +//! # } +//! # use somewhere::{Yikes, SpawnedSubsystem}; +//! # use polkadot_overseer_gen as support_crate; +//! # +//! # struct FooSubsystem; +//! # +//! impl support_crate::Subsystem for FooSubsystem //! where -//! Context: somewhere::FooSubsystemTrait, +//! Context: somewhere::FooContextTrait, //! Context: support_crate::SubsystemContext, //! ::Sender: somewhere::FooSenderTrait, //! ::Sender: somewhere::FooSenderTrait, //! { +//! fn start(self, context: Context) -> SpawnedSubsystem { +//! // .. +//! # let _ = context; +//! # unimplemented!() +//! } //! } //! ``` //! diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 4c6409651936..24d0ce413f8b 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -66,6 +66,7 @@ pub use polkadot_overseer_gen_proc_macro::{contextbounds, overlord, subsystem}; pub use gum; #[doc(hidden)] pub use metered; + #[doc(hidden)] pub use polkadot_node_primitives::SpawnNamed; From f473178fcaf650e49398d2d3321cb874f174e7c3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 11:12:40 +0200 Subject: [PATCH 125/139] can't teach hunspell `clonabl`, so drop that --- .../overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index 26e9be8914f2..bd1569bc3694 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -263,7 +263,7 @@ pub(crate) fn impl_wrapper_enum(wrapper: &Ident, message_types: &[Path]) -> Resu Ok(ts) } -/// Create the clonable subsystem sender type and implements `trait SubsystemSender` +/// Create the subsystem sender type and implements `trait SubsystemSender` /// for the `#outgoing_wrappers: From` with the proper associated types. pub(crate) fn impl_subsystem_sender( support_crate: &Path, From 0b90f2c8b223c193b33081a055532ac4f28f8072 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 11:15:38 +0200 Subject: [PATCH 126/139] fix typo, causing a rustc panic --- utils/staking-miner/src/rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/staking-miner/src/rpc.rs b/utils/staking-miner/src/rpc.rs index a4a366dc2a54..b609e197b49d 100644 --- a/utils/staking-miner/src/rpc.rs +++ b/utils/staking-miner/src/rpc.rs @@ -64,7 +64,7 @@ pub trait RpcApi { at: Option<&Hash>, ) -> RpcResult>; - /// Dry run an extrinsic at a given block. Return SCALE encoded [`sp_runtine::ApplyExtrinsicResult`]. + /// Dry run an extrinsic at a given block. Return SCALE encoded [`sp_runtime::ApplyExtrinsicResult`]. #[method(name = "system_dryRun")] async fn dry_run(&self, extrinsic: &Bytes, at: Option) -> RpcResult; From e79313b807569017b8ccb4e465f5fdd263cd1979 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 11:24:14 +0200 Subject: [PATCH 127/139] chore: doc fixes --- core-primitives/src/lib.rs | 2 +- node/metrics/src/lib.rs | 2 +- node/overseer/overseer-gen/src/lib.rs | 2 +- parachain/src/primitives.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core-primitives/src/lib.rs b/core-primitives/src/lib.rs index 4fdfe146c566..bd0256dc7c9e 100644 --- a/core-primitives/src/lib.rs +++ b/core-primitives/src/lib.rs @@ -60,7 +60,7 @@ pub type ChainId = u32; /// A hash of some data used by the relay chain. pub type Hash = sp_core::H256; -/// Unit type wrapper around [`Hash`] that represents a candidate hash. +/// Unit type wrapper around [`type@Hash`] that represents a candidate hash. /// /// This type is produced by [`CandidateReceipt::hash`]. /// diff --git a/node/metrics/src/lib.rs b/node/metrics/src/lib.rs index d648304ae751..0390788dde17 100644 --- a/node/metrics/src/lib.rs +++ b/node/metrics/src/lib.rs @@ -41,7 +41,7 @@ pub fn logger_hook() -> impl FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Con |_logger_builder, _config| {} } -/// This module reexports Prometheus types and defines the [`Metrics`] trait. +/// This module reexports Prometheus types and defines the [`Metrics`](metrics::Metrics) trait. pub mod metrics { /// Reexport Substrate Prometheus types. pub use substrate_prometheus_endpoint as prometheus; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 24d0ce413f8b..e0ec74793501 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -102,7 +102,7 @@ use std::fmt; #[cfg(test)] mod tests; -/// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. +/// A type of messages that are sent from a [`Subsystem`] to the declared overseer. /// /// Used to launch jobs. pub enum ToOverseer { diff --git a/parachain/src/primitives.rs b/parachain/src/primitives.rs index b7de1d0ada33..54c67cdf1a05 100644 --- a/parachain/src/primitives.rs +++ b/parachain/src/primitives.rs @@ -65,7 +65,7 @@ impl ValidationCode { } } -/// Unit type wrapper around [`Hash`] that represents a validation code hash. +/// Unit type wrapper around [`type@Hash`] that represents a validation code hash. /// /// This type is produced by [`ValidationCode::hash`]. /// From 42fa7be305f4a797c8f937a3a271f3eee67c77fd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 15:07:42 +0200 Subject: [PATCH 128/139] dev deps cause a false positive for deny(unused_crate_dependencies) --- node/overseer/overseer-gen/proc-macro/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 1da58ff5ee2d..7845917cfbab 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -#![deny(unused_crate_dependencies)] - use proc_macro2::{Ident, Span, TokenStream}; use syn::{parse_quote, spanned::Spanned, Path}; From 68b4621a0a483f34c910cce642781356135c3fc2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 11 May 2022 18:23:22 +0200 Subject: [PATCH 129/139] warnings --- node/core/backing/src/lib.rs | 2 +- node/core/backing/src/tests.rs | 5 ++++- node/network/bridge/src/lib.rs | 3 +-- node/network/bridge/src/tests.rs | 4 ++-- node/network/collator-protocol/src/lib.rs | 4 +--- node/network/statement-distribution/src/tests.rs | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index a7e3534071a0..c6c1a76cee61 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -42,7 +42,7 @@ use polkadot_node_subsystem::{ ProvisionableData, ProvisionerMessage, RuntimeApiRequest, StatementDistributionMessage, }, overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, PerLeafSpan, SpawnedSubsystem, - Stage, SubsystemContext, SubsystemError, SubsystemSender, + Stage, SubsystemError, }; use polkadot_node_subsystem_util::{ self as util, request_from_runtime, request_session_index_for_child, request_validator_groups, diff --git a/node/core/backing/src/tests.rs b/node/core/backing/src/tests.rs index e60d5112be1c..e7fbdf6a4e13 100644 --- a/node/core/backing/src/tests.rs +++ b/node/core/backing/src/tests.rs @@ -23,7 +23,10 @@ use assert_matches::assert_matches; use futures::{future, Future}; use polkadot_node_primitives::{BlockData, InvalidCandidate}; use polkadot_node_subsystem::{ - messages::{CollatorProtocolMessage, RuntimeApiMessage, RuntimeApiRequest, ValidationFailed}, + messages::{ + AllMessages, CollatorProtocolMessage, RuntimeApiMessage, RuntimeApiRequest, + ValidationFailed, + }, ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, LeafStatus, OverseerSignal, }; use polkadot_node_subsystem_test_helpers as test_helpers; diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 65a828cbb300..e11d3500487a 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -44,8 +44,7 @@ use polkadot_node_subsystem::{ }, overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, }; -use polkadot_node_subsystem_util::metrics::{self, prometheus}; -use polkadot_overseer::gen::{OverseerError, Subsystem}; +use polkadot_overseer::gen::OverseerError; use polkadot_primitives::v2::{AuthorityDiscoveryId, BlockNumber, Hash, ValidatorIndex}; /// Peer set info for network initialization. diff --git a/node/network/bridge/src/tests.rs b/node/network/bridge/src/tests.rs index 62291be8c662..09c7f270c11f 100644 --- a/node/network/bridge/src/tests.rs +++ b/node/network/bridge/src/tests.rs @@ -34,8 +34,8 @@ use polkadot_node_network_protocol::{ use polkadot_node_subsystem::{ jaeger, messages::{ - ApprovalDistributionMessage, BitfieldDistributionMessage, GossipSupportMessage, - StatementDistributionMessage, + AllMessages, ApprovalDistributionMessage, BitfieldDistributionMessage, + GossipSupportMessage, StatementDistributionMessage, }, ActiveLeavesUpdate, FromOverseer, LeafStatus, OverseerSignal, }; diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 4a3998f94a79..bdf8904b7d07 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -34,9 +34,7 @@ use polkadot_node_network_protocol::{ use polkadot_primitives::v2::CollatorPair; use polkadot_node_subsystem::{ - errors::SubsystemError, - messages::{CollatorProtocolMessage, NetworkBridgeMessage}, - overseer, SpawnedSubsystem, SubsystemSender, + errors::SubsystemError, messages::NetworkBridgeMessage, overseer, SpawnedSubsystem, }; mod error; diff --git a/node/network/statement-distribution/src/tests.rs b/node/network/statement-distribution/src/tests.rs index 49a8e5cd1426..a0342fea2eba 100644 --- a/node/network/statement-distribution/src/tests.rs +++ b/node/network/statement-distribution/src/tests.rs @@ -29,7 +29,7 @@ use polkadot_node_network_protocol::{ use polkadot_node_primitives::{Statement, UncheckedSignedFullStatement}; use polkadot_node_subsystem::{ jaeger, - messages::{network_bridge_event, RuntimeApiMessage, RuntimeApiRequest}, + messages::{network_bridge_event, AllMessages, RuntimeApiMessage, RuntimeApiRequest}, ActivatedLeaf, LeafStatus, }; use polkadot_node_subsystem_test_helpers::mock::make_ferdie_keystore; From ac2a89d2e4266524e35086be80ce496e818ff079 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 08:57:08 +0200 Subject: [PATCH 130/139] fix rustdoc complaint --- node/primitives/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/primitives/src/lib.rs b/node/primitives/src/lib.rs index 6e28b1d34d9f..c203e560647d 100644 --- a/node/primitives/src/lib.rs +++ b/node/primitives/src/lib.rs @@ -86,7 +86,7 @@ pub const MAX_FINALITY_LAG: u32 = 500; /// We are not using `NonZeroU32` here because `expect` and `unwrap` are not yet const, so global /// constants of `SessionWindowSize` would require `lazy_static` in that case. /// -/// See: https://github.com/rust-lang/rust/issues/67441 +/// See: #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] pub struct SessionWindowSize(SessionIndex); From 786965bf82ac730c2c78e939e8618730e1a785ee Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 09:04:42 +0200 Subject: [PATCH 131/139] disable doctests, broken in current nightly See https://github.com/paritytech/ci_cd/issues/427 --- node/overseer/overseer-gen/proc-macro/src/subsystem.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 2324217f449d..eda12f732f31 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -19,7 +19,7 @@ //! //! ## Implement `trait Subsystem` via `subsystem` //! -//! ```no_run +//! ```ignore //! # use polkadot_overseer_gen_proc_macro::subsystem; //! # mod somewhere { //! # use polkadot_overseer_gen_proc_macro::overlord; @@ -70,7 +70,7 @@ //! //! expands to //! -//! ```no_run +//! ```ignore //! # use polkadot_overseer_gen_proc_macro::subsystem; //! # mod somewhere { //! # use polkadot_overseer_gen_proc_macro::overlord; From 0ba7af47e15c88fe5f4eb1f4bcc29d6f3f75ece4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 15:35:14 +0200 Subject: [PATCH 132/139] remove obsolete comment --- .../availability-distribution/src/requester/fetch_task/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index a7684f600025..ff1e5e6a73f3 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -84,7 +84,7 @@ enum FetchedState { /// Messages sent from `FetchTask`s to be handled/forwarded. pub enum FromFetchTask { /// Message to other subsystem. - Message(overseer::AvailabilityDistributionOutgoingMessages), // FIXME TODO + Message(overseer::AvailabilityDistributionOutgoingMessages), /// Concluded with result. /// From 7db9f3332678affee6f62655cc50934a522624f3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 15:40:39 +0200 Subject: [PATCH 133/139] remove some stale or known todos that don't really affect anything They are really nice to haves --- node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs | 2 +- .../overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs | 2 +- .../overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs | 1 - node/overseer/overseer-gen/proc-macro/src/subsystem.rs | 3 ++- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 24b3d4be0dd2..b3bb74665687 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -37,7 +37,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> proc_macro2::TokenStr S: #support_crate ::SpawnNamed, }; // TODO add `where ..` clauses for baggage types - // TODO https://github.com/paritytech/polkadot/issues/3427 + // TODO let consumes = &info.consumes_without_wip(); let consumes_variant = &info.variant_names_without_wip(); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index bd1569bc3694..25e7f35a927b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -221,7 +221,7 @@ fn to_variant(path: &Path, span: Span) -> Result { /// Converts the outgoing message types to variants. /// -/// TODO: Commonly this is `${X}Message` becomes `${X}OutgoingMessages::${X}Message` +/// Note: Commonly this is `${X}Message` becomes `${X}OutgoingMessages::${X}Message` /// where for `AllMessages` it would be `AllMessages::${X}`. fn to_variants(message_types: &[Path], span: Span) -> Result> { let variants: Vec<_> = diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs index 38c26a8cf377..179a4ada06eb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_overseer_struct.rs @@ -476,7 +476,6 @@ impl OverseerGuts { return Err(Error::new(span, "Must provide exactly one consuming message type")) }; - // TODO move the send and consumes check here subsystems.push(SubSysField { name: ident, generic, diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index eda12f732f31..6bac39df1f0a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -202,7 +202,8 @@ pub(crate) fn impl_subsystem_context_trait_bounds( // always prefer the direct usage, if it's not there, let's see if there is // a `prefix=*` provided. Either is ok. - // TODO: technically this is two different things: + + // Technically this is two different things: // The place where the `#[overlord]` is annotated is where all `trait *SenderTrait` and // `trait *ContextTrait` types exist. // The other usage is the true support crate `polkadot-overseer-gen`, where the static ones From 863adfdf215c4e5c62c5f58bb3781ace7004e5cc Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 15:42:30 +0200 Subject: [PATCH 134/139] Update node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs Co-authored-by: Tsvetomir Dimitrov --- .../overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs index 25e7f35a927b..9bb051a38703 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_sender.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify From 481170335dc4903e9f54e96f3000c49b685a121c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 15:42:38 +0200 Subject: [PATCH 135/139] Update node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs Co-authored-by: Tsvetomir Dimitrov --- .../overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs index 6423f0f098fc..ec308a6b804f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse/parse_subsystem_attr.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify From 0c1093fe9c841dc281d99c1501fa9161023be569 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 15:42:46 +0200 Subject: [PATCH 136/139] Update node/network/bridge/src/metrics.rs Co-authored-by: Tsvetomir Dimitrov --- node/network/bridge/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/network/bridge/src/metrics.rs b/node/network/bridge/src/metrics.rs index b648a01a9248..4ecdd5bd6f13 100644 --- a/node/network/bridge/src/metrics.rs +++ b/node/network/bridge/src/metrics.rs @@ -1,4 +1,4 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify From 56aff7c23c91d9583796b9418dd78ae0c3faecf9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 15:42:54 +0200 Subject: [PATCH 137/139] Update node/overseer/overseer-gen/proc-macro/src/subsystem.rs Co-authored-by: Tsvetomir Dimitrov --- node/overseer/overseer-gen/proc-macro/src/subsystem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs index 6bac39df1f0a..7b5523e4b5c8 100644 --- a/node/overseer/overseer-gen/proc-macro/src/subsystem.rs +++ b/node/overseer/overseer-gen/proc-macro/src/subsystem.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify From 8f76d6896d21b87a83157b8fe1c1e85393871947 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 15:48:01 +0200 Subject: [PATCH 138/139] improve README.md on what subsystem and contextbounds proc macros do --- node/overseer/overseer-gen/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/overseer/overseer-gen/README.md b/node/overseer/overseer-gen/README.md index 21227049d731..ebce568ce9a5 100644 --- a/node/overseer/overseer-gen/README.md +++ b/node/overseer/overseer-gen/README.md @@ -23,6 +23,8 @@ declarative. being consumed by that particular subsystem. Each of those subsystems is required to implement the subsystem trait with the correct trait bounds. Commonly this is achieved by using `#[subsystem]` and `#[contextbounds]` macro. + * `#[contextbounds(Foo, error=Yikes, prefix=wherethetraitsat)]` can applied to `impl`-blocks and `fn`-blocks. It will add additional trait bounds for the generic `Context` with `Context: FooContextTrait` for `::Sender: FooSenderTrait` besides a few more. Note that `Foo` here references the name of the subsystem as declared in `#[overlord(..)]` macro. + * `#[subsystem(Foo, error=Yikes, prefix=wherethetraitsat)]` is a extension to the above, implementing `trait Subsystem`. * `error=` tells the overseer to use the user provided error type, if not provided a builtin one is used. Note that this is the one error type used throughout all calls, so make sure it does impl `From` for all other error types `E` that are relevant to your application. * `event=` declares an external event type, that injects certain events From a4ab390f34485b6340c192399d7e89dcc172cac0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 12 May 2022 16:19:51 +0200 Subject: [PATCH 139/139] Update node/overseer/overseer-gen/proc-macro/src/overseer.rs Co-authored-by: Tsvetomir Dimitrov --- node/overseer/overseer-gen/proc-macro/src/overseer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/overseer.rs b/node/overseer/overseer-gen/proc-macro/src/overseer.rs index 99a74d0c476b..127120337ae0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/overseer.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify