Merge branch 'master' into minimal-node-retry-conn-to-external-rpc

paritytech · Aug 30, 2024 · 0e831e7 · 0e831e7
2 parents 8e18ede + 09035a7
commit 0e831e7
Show file tree

Hide file tree

Showing 37 changed files with 420 additions and 274 deletions.
diff --git a/.github/actions/set-up-mac/README.md b/.github/actions/set-up-mac/README.md
@@ -0,0 +1,15 @@
+# How to use
+
+```yml
+  set-image:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+      - name: Install dependencies
+        uses: ./.github/actions/set-up-mac
+        with:
+          IMAGE: ${{ steps.set-image.outputs.IMAGE }}
+```
diff --git a/.github/actions/set-up-mac/action.yml b/.github/actions/set-up-mac/action.yml
@@ -0,0 +1,43 @@
+name: "Set up rust on mac"
+description: "Install the required tools for Mac runners"
+inputs:
+  IMAGE:
+    description: "Rust docker image"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Install with Hombrew
+      shell: bash
+      run: brew install protobuf rustup openssl pkg-config zlib xz zstd llvm jq curl gcc make cmake
+    - name: Set version
+      shell: bash
+      run: |
+        VERSION=$(echo $IMAGE | sed -E 's/.*:bullseye-([^-]+)-.*/\1/')
+        echo $VERSION
+        echo "VERSION=$VERSION" >> $GITHUB_ENV
+        NIGHTLY=$(echo $IMAGE | sed -E 's/.*([0-9]{4}-[0-9]{2}-[0-9]{2}).*/\1/')
+        echo $NIGHTLY
+        echo "NIGHTLY=$NIGHTLY" >> $GITHUB_ENV
+      env:
+        IMAGE: ${{ inputs.IMAGE }}
+
+    - name: Install rustup
+      shell: bash
+      run: |
+        rustup-init -y
+        rustup install $VERSION
+        rustup default $VERSION
+        rustup toolchain install "nightly-${NIGHTLY}"
+
+    - name: MacOS Deps
+      shell: bash
+      run: |
+        rustup target add wasm32-unknown-unknown --toolchain $VERSION
+        rustup component add rust-src rustfmt clippy --toolchain $VERSION
+
+    - name: Check Rust
+      shell: bash
+      run: |
+        rustup show
+        rustup +nightly show
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -810,6 +810,7 @@ is-terminal = { version = "0.4.9" }
 is_executable = { version = "1.0.1" }
 isahc = { version = "1.2" }
 itertools = { version = "0.11" }
+jemalloc_pprof = { version = "0.4" }
 jobserver = { version = "0.1.26" }
 jsonpath_lib = { version = "0.3" }
 jsonrpsee = { version = "0.24.3" }

diff --git a/polkadot/node/collation-generation/src/lib.rs b/polkadot/node/collation-generation/src/lib.rs
@@ -45,7 +45,7 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_util::{
 	request_async_backing_params, request_availability_cores, request_para_backing_state,
 	request_persisted_validation_data, request_validation_code, request_validation_code_hash,
-	request_validators, vstaging::fetch_claim_queue,
+	request_validators, runtime::fetch_claim_queue,
 };
 use polkadot_primitives::{
 	collator_signature_payload, CandidateCommitments, CandidateDescriptor, CandidateReceipt,

diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs
@@ -100,9 +100,9 @@ use polkadot_node_subsystem_util::{
 	executor_params_at_relay_parent, request_from_runtime, request_session_index_for_child,
 	request_validator_groups, request_validators,
 	runtime::{
-		self, prospective_parachains_mode, request_min_backing_votes, ProspectiveParachainsMode,
+		self, fetch_claim_queue, prospective_parachains_mode, request_min_backing_votes,
+		ClaimQueueSnapshot, ProspectiveParachainsMode,
 	},
-	vstaging::{fetch_claim_queue, ClaimQueueSnapshot},
 	Validator,
 };
 use polkadot_primitives::{
@@ -121,7 +121,7 @@ use polkadot_statement_table::{
 	Config as TableConfig, Context as TableContextTrait, Table,
 };
 use sp_keystore::KeystorePtr;
-use util::{runtime::request_node_features, vstaging::get_disabled_validators_with_fallback};
+use util::runtime::{get_disabled_validators_with_fallback, request_node_features};
 
 mod error;
 

diff --git a/polkadot/node/core/prospective-parachains/src/lib.rs b/polkadot/node/core/prospective-parachains/src/lib.rs
@@ -46,8 +46,7 @@ use polkadot_node_subsystem_util::{
 	backing_implicit_view::{BlockInfoProspectiveParachains as BlockInfo, View as ImplicitView},
 	inclusion_emulator::{Constraints, RelayChainBlockInfo},
 	request_session_index_for_child,
-	runtime::{prospective_parachains_mode, ProspectiveParachainsMode},
-	vstaging::fetch_claim_queue,
+	runtime::{fetch_claim_queue, prospective_parachains_mode, ProspectiveParachainsMode},
 };
 use polkadot_primitives::{
 	async_backing::CandidatePendingAvailability, BlockNumber, CandidateHash,

diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs
@@ -48,10 +48,9 @@ use polkadot_node_subsystem_util::{
 	backing_implicit_view::View as ImplicitView,
 	reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL},
 	runtime::{
-		get_availability_cores, get_group_rotation_info, prospective_parachains_mode,
-		ProspectiveParachainsMode, RuntimeInfo,
+		fetch_claim_queue, get_availability_cores, get_group_rotation_info,
+		prospective_parachains_mode, ProspectiveParachainsMode, RuntimeInfo,
 	},
-	vstaging::fetch_claim_queue,
 	TimeoutExt,
 };
 use polkadot_primitives::{

diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs
@@ -50,8 +50,7 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_util::{
 	backing_implicit_view::View as ImplicitView,
 	reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL},
-	runtime::{prospective_parachains_mode, ProspectiveParachainsMode},
-	vstaging::fetch_claim_queue,
+	runtime::{fetch_claim_queue, prospective_parachains_mode, ProspectiveParachainsMode},
 };
 use polkadot_primitives::{
 	CandidateHash, CollatorId, CoreState, Hash, HeadData, Id as ParaId, OccupiedCoreAssumption,

diff --git a/polkadot/node/network/statement-distribution/src/v2/mod.rs b/polkadot/node/network/statement-distribution/src/v2/mod.rs
@@ -45,8 +45,9 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_util::{
 	backing_implicit_view::View as ImplicitView,
 	reputation::ReputationAggregator,
-	runtime::{request_min_backing_votes, ProspectiveParachainsMode},
-	vstaging::{fetch_claim_queue, ClaimQueueSnapshot},
+	runtime::{
+		fetch_claim_queue, request_min_backing_votes, ClaimQueueSnapshot, ProspectiveParachainsMode,
+	},
 };
 use polkadot_primitives::{
 	AuthorityDiscoveryId, CandidateHash, CompactStatement, CoreIndex, CoreState, GroupIndex,
@@ -570,7 +571,7 @@ pub(crate) async fn handle_active_leaves_update<Context>(
 
 	for new_relay_parent in new_relay_parents.iter().cloned() {
 		let disabled_validators: HashSet<_> =
-			polkadot_node_subsystem_util::vstaging::get_disabled_validators_with_fallback(
+			polkadot_node_subsystem_util::runtime::get_disabled_validators_with_fallback(
 				ctx.sender(),
 				new_relay_parent,
 			)

diff --git a/polkadot/node/network/statement-distribution/src/v2/statement_store.rs b/polkadot/node/network/statement-distribution/src/v2/statement_store.rs
@@ -292,7 +292,7 @@ impl GroupStatements {
 mod tests {
 	use super::*;
 
-	use polkadot_primitives::v7::{Hash, SigningContext, ValidatorPair};
+	use polkadot_primitives::{Hash, SigningContext, ValidatorPair};
 	use sp_application_crypto::Pair as PairT;
 
 	#[test]

diff --git a/polkadot/node/service/src/chain_spec.rs b/polkadot/node/service/src/chain_spec.rs
@@ -25,7 +25,7 @@ use sp_consensus_babe::AuthorityId as BabeId;
 use sp_consensus_beefy::ecdsa_crypto::AuthorityId as BeefyId;
 
 #[cfg(feature = "westend-native")]
-use polkadot_primitives::vstaging::SchedulerParams;
+use polkadot_primitives::SchedulerParams;
 #[cfg(feature = "rococo-native")]
 use rococo_runtime as rococo;
 use sc_chain_spec::ChainSpecExtension;

diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml
@@ -19,7 +19,11 @@ path = "src/cli/subsystem-bench.rs"
 # Prevent rustdoc error. Already documented from top-level Cargo.toml.
 doc = false
 
+
 [dependencies]
+tikv-jemallocator = { features = ["profiling", "unprefixed_malloc_on_supported_platforms"], workspace = true, optional = true }
+jemalloc_pprof = { workspace = true, optional = true }
+polkadot-service = { workspace = true, default-features = true }
 polkadot-node-subsystem = { workspace = true, default-features = true }
 polkadot-node-subsystem-util = { workspace = true, default-features = true }
 polkadot-node-subsystem-types = { workspace = true, default-features = true }
@@ -93,3 +97,7 @@ strum = { features = ["derive"], workspace = true, default-features = true }
 
 [features]
 default = []
+memprofile = [
+	"dep:jemalloc_pprof",
+	"dep:tikv-jemallocator",
+]
diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md
@@ -260,6 +260,41 @@ This file is best interpreted with `cg_annotate --auto=yes cachegrind.out.<pid>`
 
 For finer profiling of cache misses, better use `perf` on a bare-metal machine.
 
+### Profile memory usage using jemalloc
+
+Bellow you can find instructions how to setup and run profiling with jemalloc, this is complementary
+with using other memory profiling tools like: <https://github.com/koute/bytehound?tab=readme-ov-file#basic-usage>.
+
+#### Prerequisites
+
+Install tooling with:
+
+```
+sudo apt install libjemalloc-dev graphviz
+```
+
+#### Generate memory usage snapshots
+
+Memory usage can be profiled by running any subsystem benchmark with `--features memprofile`, e.g:
+
+```
+RUSTFLAGS=-g cargo run -p polkadot-subsystem-bench --release --features memprofile -- polkadot/node/subsystem-bench/examples/approvals_throughput.yaml
+```
+
+#### Interpret the results
+
+After the benchmark ran the memory usage snapshots can be found in `/tmp/subsystem-bench*`, to extract the information
+from a snapshot you can use `jeprof` like this:
+
+```
+jeprof --text PATH_TO_EXECUTABLE_WITH_DEBUG_SYMBOLS /tmp/subsystem-bench.1222895.199.i199.heap > statistics.txt
+```
+
+Useful links:
+
+- Tutorial: <https://www.magiroux.com/rust-jemalloc-profiling/>
+- Jemalloc configuration options: <https://jemalloc.net/jemalloc.3.html>
+
 ## Create new test objectives
 
 This tool is intended to make it easy to write new test objectives that focus individual subsystems,

diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs
@@ -182,6 +182,17 @@ impl BenchCli {
 	}
 }
 
+#[cfg(feature = "memprofile")]
+#[global_allocator]
+static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+#[cfg(feature = "memprofile")]
+#[allow(non_upper_case_globals)]
+#[export_name = "malloc_conf"]
+// See https://jemalloc.net/jemalloc.3.html for more information on the configuration options.
+pub static malloc_conf: &[u8] =
+	b"prof:true,prof_active:true,lg_prof_interval:30,lg_prof_sample:21,prof_prefix:/tmp/subsystem-bench\0";
+
 fn main() -> eyre::Result<()> {
 	color_eyre::install()?;
 	sp_tracing::try_init_simple();

diff --git a/polkadot/node/subsystem-util/src/lib.rs b/polkadot/node/subsystem-util/src/lib.rs
@@ -49,6 +49,7 @@ use polkadot_primitives::{
 	ValidatorSignature,
 };
 pub use rand;
+use runtime::get_disabled_validators_with_fallback;
 use sp_application_crypto::AppCrypto;
 use sp_core::ByteArray;
 use sp_keystore::{Error as KeystoreError, KeystorePtr};
@@ -57,7 +58,6 @@ use std::{
 	time::Duration,
 };
 use thiserror::Error;
-use vstaging::get_disabled_validators_with_fallback;
 
 pub use determine_new_blocks::determine_new_blocks;
 pub use metered;