feat: Indexing configs

CHANGELOG.md

@@ -5,6 +5,11 @@
 - ci: disable pr close workflow
 - ci: add ci verification for detecting genesis changes and config hashes
 - feat: better management of custom configurations for genesis assets
+- feat: unification of the DA interface
+- feat: use resolver 2 for cargo in the workspace
+- upgrade: restructure code for rust latest version
+- upgrade: bump rustc nightly version to 1.74 date
+- feat: bump starknet-core to 0.6.0 and remove InvokeV0
 - fix: estimate_fee should make sure all transaction have a version being
   2^128 + 1 or 2^128+2 depending on the tx type
 - fix: initial_gas set to max_fee and fixed fee not being charged when max_fee=0

Cargo.toml

@@ -165,9 +165,9 @@ clap = { version = "4.4.2", default-features = false }
 futures = { version = "0.3.28", default-features = false }
 futures-timer = { version = "3.0.2", default-features = false }
 md5 = { version = "0.7.0", default-features = false }
-reqwest = { version = "0.11.18", default-features = false }
-serde = { version = "1.0.185", default-features = false }
-serde_json = { version = "1.0.105", default-features = false }
+reqwest = { version = "0.11.20", default-features = false }
+serde = { version = "1.0.188", default-features = false }
+serde_json = { version = "1.0.106", default-features = false }
 serde_with = { version = "2.3.3", default-features = false }
 bitvec = { version = "1", default-features = false }
 thiserror = "1.0.48"

README.md

@@ -141,16 +141,6 @@ production environment, the relative changes and trends over time are what we
 focus on. This way, we can ensure that Madara is always improving, and that we
 maintain a high standard of performance as the project evolves.
 
-One can use flamegraph-rs to generate flamegraphs and look for the performance
-bottlenecks of the system by running the following :
-
-```bash
-flamegraph --root --open  -- ./target/release/madara --dev --pool-limit=100000 --pool-kbytes=500000 --rpc-methods=unsafe --rpc-cors=all --in-peers=0 --out-peers=1 --no-telemetry
-```
-
-In parallel to that, run `npm run test` within the `benchmarking` folder. Once
-you stop the node, the flamegraph will open in your browser.
-
 ## 🌐 Connect to the dev webapp
 
 Once your Madara node is up and running, you can connect to our Dev Frontend App

crates/node/src/chain_spec.rs

@@ -145,8 +145,9 @@ pub fn local_testnet_config(madara_path: PathBuf) -> Result<ChainSpec, String> {
 
 fn load_genesis(madara_path: PathBuf) -> GenesisLoader {
     let madara_path = madara_path.to_str().unwrap().to_string();
-    let genesis_path = madara_path.clone() + "/genesis-assets/genesis.json";
-    let genesis = utils::read_file_to_string(genesis_path).expect("Failed to read genesis file");
+    let genesis_path = madara_path.clone() + "/configs/genesis-assets/genesis.json";
+    let genesis = utils::read_file_to_string(genesis_path)
+        .expect("Failed to read genesis file. Please run `madara setup` before opening an issue.");
     let mut genesis_loader: GenesisLoader = serde_json::from_str(&genesis).expect("Failed loading genesis");
     genesis_loader.set_madara_path(madara_path);
     genesis_loader
@@ -159,12 +160,7 @@ fn testnet_genesis(
     initial_authorities: Vec<(AuraId, GrandpaId)>,
     _enable_println: bool,
 ) -> GenesisConfig {
-    let genesis = madara_path.clone() + "/configs/genesis-assets/genesis.json";
-    let genesis = utils::read_file_to_string(genesis)
-        .expect("Failed to read genesis file. Please run `madara setup` before opening an issue.");
-    let mut genesis: GenesisLoader = serde_json::from_str(&genesis).expect("Failed loading genesis");
-    genesis.set_madara_path(madara_path);
-    let starknet_genesis: madara_runtime::pallet_starknet::GenesisConfig<_> = genesis.into();
+    let starknet_genesis_config: madara_runtime::pallet_starknet::GenesisConfig<_> = genesis_loader.into();
 
     GenesisConfig {
         system: SystemConfig {

crates/node/src/cli.rs

@@ -36,17 +36,17 @@ pub struct Cli {
 
     #[clap(flatten)]
     pub setup: SetupCmd,
-
-    /// Choose sealing method.
-    #[arg(long, value_enum, ignore_case = true)]
-    pub sealing: Option<Sealing>,
 }
 
-#[derive(Debug, clap::Args)]
+#[derive(Clone, Debug, clap::Args)]
 pub struct ExtendedRunCmd {
     #[clap(flatten)]
     pub run_cmd: RunCmd,
 
+    /// Choose a supported DA Layer
+    #[clap(long)]
+    pub da_layer: Option<DaLayer>,
+
     /// Load a custom chain-spec from an url
     /// If you want to load a chain spec that is present in your filesystem, use `--chain=<PATH>`
     #[clap(long, conflicts_with = "testnet")]
@@ -58,9 +58,13 @@ pub struct ExtendedRunCmd {
     #[clap(long, default_value = get_default_madara_path())]
     pub madara_path: Option<PathBuf>,
 
+    /// Choose sealing method.
+    #[arg(long, value_enum, ignore_case = true)]
+    pub sealing: Option<Sealing>,
+
     /// Choose a supported testnet chain which will load some default values
     /// The testnets will allways be fetched when this flag is passed to search for updates
-    #[clap(long, conflicts_with = "fetch_chain_spec")]
+    #[clap(long, conflicts_with = "fetch_chain_spec", conflicts_with = "chain")]
     pub testnet: Option<Testnet>,
 }
 

crates/node/src/command.rs

@@ -37,7 +37,7 @@ impl SubstrateCli for Cli {
     fn load_spec(&self, id: &str) -> Result<Box<dyn sc_service::ChainSpec>, String> {
         Ok(match id {
             "dev" => {
-                let enable_manual_seal = self.sealing.map(|_| true);
+                let enable_manual_seal = self.run.sealing.map(|_| true);
                 Box::new(chain_spec::development_config(
                     enable_manual_seal,
                     self.run.madara_path.clone().expect("Failed retrieving madara_path"),
@@ -56,6 +56,7 @@ impl SubstrateCli for Cli {
 }
 
 fn set_dev_environment(cli: &mut Cli) {
+    println!("Setting up dev environment");
     // create a reproducible dev environment
     cli.run.run_cmd.shared_params.dev = false;
     cli.run.run_cmd.shared_params.chain = Some("dev".to_string());
@@ -74,13 +75,12 @@ fn set_testnet(cli: &mut Cli, testnet: Testnet) -> Result<(), String> {
     let madara_path = cli.run.madara_path.clone().expect("Failed retrieving madara_path").to_str().unwrap().to_string();
     let local_path = utils::get_project_path();
 
-    match (testnet, local_path) {
-        (Testnet::Sharingan, Ok(ref src_path)) => {
+    if testnet == Testnet::Sharingan {
+        if let Ok(ref src_path) = local_path {
             let src_path = src_path.clone() + "/configs/chain-specs/testnet-sharingan-raw.json";
             utils::copy_from_filesystem(src_path, madara_path.clone() + "/chain-specs")?;
             cli.run.run_cmd.shared_params.chain = Some(madara_path + "/chain-specs/testnet-sharingan-raw.json");
-        }
-        (Testnet::Sharingan, _) => {
+        } else {
             utils::fetch_from_url(
                 constants::SHARINGAN_CHAIN_SPEC_URL.to_string(),
                 madara_path.clone() + "/configs/chain-specs/",
@@ -107,40 +107,35 @@ fn set_chain_spec(cli: &mut Cli, chain_spec_url: String) -> Result<(), String> {
     Ok(())
 }
 
-fn fetch_madara_configs(cli: &mut Cli) -> Result<(), String> {
+fn fetch_madara_configs(cli: &Cli) -> Result<(), String> {
     let madara_path = cli.run.madara_path.clone().expect("Failed retrieving madara_path").to_str().unwrap().to_string();
     let local_path = utils::get_project_path();
 
-    match (cli.setup.fetch_madara_configs.clone(), local_path) {
-        // TODO match string for default configs value
-        (_, Ok(ref src_path)) => {
-            let index_path = src_path.clone() + "/configs/index.json";
-            utils::copy_from_filesystem(index_path, madara_path.clone() + "/configs")?;
-
-            let madara_configs: configs::Configs =
-                serde_json::from_str(&utils::read_file_to_string(madara_path.clone() + "/configs/index.json")?)
-                    .expect("Failed to load madara configs");
-            for asset in madara_configs.genesis_assets {
-                let src_path = src_path.clone() + "/configs/genesis-assets/" + &asset.name;
-                utils::copy_from_filesystem(src_path, madara_path.clone() + "/configs/genesis-assets")?;
-            }
+    if let Ok(ref src_path) = local_path {
+        let index_path = src_path.clone() + "/configs/index.json";
+        utils::copy_from_filesystem(index_path, madara_path.clone() + "/configs")?;
+
+        let madara_configs: configs::Configs =
+            serde_json::from_str(&utils::read_file_to_string(madara_path.clone() + "/configs/index.json")?)
+                .expect("Failed to load madara configs");
+        for asset in madara_configs.genesis_assets {
+            let src_path = src_path.clone() + "/configs/genesis-assets/" + &asset.name;
+            utils::copy_from_filesystem(src_path, madara_path.clone() + "/configs/genesis-assets")?;
         }
-        (Some(configs_url), _) => {
-            utils::fetch_from_url(configs_url, madara_path.clone() + "/configs")?;
-
-            let madara_configs: configs::Configs =
-                serde_json::from_str(&utils::read_file_to_string(madara_path.clone() + "/configs/index.json")?)
-                    .expect("Failed to load madara configs");
-
-            for asset in madara_configs.genesis_assets {
-                configs::fetch_and_validate_file(
-                    madara_configs.remote_base_path.clone(),
-                    asset,
-                    madara_path.clone() + "/configs/genesis-assets/",
-                )?;
-            }
+    } else if let Some(configs_url) = &cli.setup.fetch_madara_configs {
+        utils::fetch_from_url(configs_url.to_string(), madara_path.clone() + "/configs")?;
+
+        let madara_configs: configs::Configs =
+            serde_json::from_str(&utils::read_file_to_string(madara_path.clone() + "/configs/index.json")?)
+                .expect("Failed to load madara configs");
+
+        for asset in madara_configs.genesis_assets {
+            configs::fetch_and_validate_file(
+                madara_configs.remote_base_path.clone(),
+                asset,
+                madara_path.clone() + "/configs/genesis-assets/",
+            )?;
         }
-        _ => {}
     }
 
     Ok(())
@@ -270,17 +265,17 @@ pub fn run() -> sc_cli::Result<()> {
             let runner = cli.create_runner(cmd)?;
             runner.sync_run(|config| cmd.run::<Block>(&config))
         }
-        Some(Subcommand::Run(_)) => {
+        Some(Subcommand::Run(cmd)) => {
             let madara_path =
                 cli.run.madara_path.clone().expect("Failed retrieving madara_path").to_str().unwrap().to_string();
 
             // Set the node_key_file for substrate in the case that it was not manually setted
-            if cli.run.run_cmd.network_params.node_key_params.node_key_file.is_none() {
+            if cmd.run_cmd.network_params.node_key_params.node_key_file.is_none() {
                 cli.run.run_cmd.network_params.node_key_params.node_key_file =
-                    Some((madara_path + "/p2p-key.ed25519").into());
+                    Some((madara_path.clone() + "/p2p-key.ed25519").into());
             }
 
-            if cli.run.run_cmd.shared_params.dev {
+            if cmd.run_cmd.shared_params.dev {
                 set_dev_environment(&mut cli);
             }
 
@@ -310,11 +305,11 @@ pub fn run() -> sc_cli::Result<()> {
 
             let runner = cli.create_runner(&cli.run.run_cmd)?;
             runner.run_node_until_exit(|config| async move {
-                service::new_full(config, cli.sealing, da_config).map_err(sc_cli::Error::Service)
+                service::new_full(config, cli.run.sealing, da_config).map_err(sc_cli::Error::Service)
             })
         }
         Some(Subcommand::Setup(_)) => {
-            fetch_madara_configs(&mut cli)?;
+            fetch_madara_configs(&cli)?;
             Ok(())
         }
         _ => Err("You need to specify some subcommand. E.g. `madara run`".into()),

docs/faucet-setup.md

@@ -17,7 +17,8 @@ Follow the steps below to setup a faucet for your local build
 Run an instance of your madara node locally
 
 ```bash
-cargo run --release -- --dev
+cargo run --release -- setup
+cargo run --release -- run --dev
 ```
 
 `--dev`: enforces a development environment needed to make testing easier for

docs/getting-started.md

@@ -16,24 +16,25 @@ rustup show
 Use Rust's native `cargo` command to build and launch the template node:
 
 ```sh
-cargo run --release -- --dev
+cargo run --release -- setup
+cargo run --release -- run --dev
 ```
 
 The node also supports to use manual seal (to produce block manually through
 RPC). This is also used by the typescript tests:
 
 ```sh
-$ cargo run --release -- --dev --sealing=manual
+$ cargo run --release -- run --dev --sealing=manual
 # Or
-$ cargo run --release -- --dev --sealing=instant
+$ cargo run --release -- run --dev --sealing=instant
 ```
 
 Log level can be specified with `-l` flag. For example, `-ldebug` will show
 debug logs. It can also be specified via the `RUST_LOG` environment variable.
 For example:
 
 ```sh
-RUSTLOG=runtime=info cargo run --release -- --dev
+RUSTLOG=runtime=info cargo run --release -- run --dev
 ```
 
 ### Cargo Build
@@ -80,7 +81,8 @@ This command will start the single-node development chain with non-persistent
 state:
 
 ```bash
-./target/release/madara --dev
+./target/release/madara setup
+./target/release/madara run --dev
 ```
 
 Purge the development chain's state:
@@ -92,7 +94,7 @@ Purge the development chain's state:
 Start the development chain with detailed logging:
 
 ```bash
-RUST_BACKTRACE=1 ./target/release/madara -ldebug --dev
+RUST_BACKTRACE=1 ./target/release/madara run -ldebug --dev
 ```
 
 > Development chain means that the state of our chain will be in a tmp folder
@@ -117,7 +119,7 @@ commands shows how to use a newly created folder as our db base path.
 $ mkdir my-chain-state
 
 // Use of that folder to store the chain state
-$ ./target/release/madara --dev --base-path ./my-chain-state/
+$ ./target/release/madara run --dev --base-path ./my-chain-state/
 
 // Check the folder structure created inside the base path after running the chain
 $ ls ./my-chain-state

docs/rpc-contribution.md

@@ -17,8 +17,9 @@ First, go ahead and clone madara on the `main` branch from
 
 There are two ways you can build madara to quickly test it:
 
-1. `cargo build --release`, which will then allow us to start madara running
-   `./target/release/madara`. This will start the sequencer WITHOUT peers.
+1. `cargo build --release`, which will then allow us to setup madara with
+   `./target/release/madara setup`, and then run it with
+   `./target/release/madara run`. This will start the sequencer WITHOUT peers.
    That's not a problem if you just want to test that your RPC method is
    accessible, and to test (de)serialization of your RPC parameters.
 

docs/sharingan-starter-pack.md

@@ -226,7 +226,8 @@ If you prefer having Madara compiled locally, you must:
    the root of Madara repository:
 
 ```bash
-./target/release/madara --testnet sharingan --telemetry-url 'wss://telemetry.madara.zone/submit 0'
+./target/release/madara setup
+./target/release/madara run --testnet sharingan --telemetry-url 'wss://telemetry.madara.zone/submit 0'
 ```
 
 This will store the data into `$HOME/.madara`.

tests/util/dev-node.ts

@@ -74,6 +74,7 @@ export async function startMadaraDevNode(
 
   const cmd = BINARY_PATH;
   const args = [
+    "run",
     withWasm ? "--execution=Wasm" : "--execution=Native", // Faster execution using native
     process.env.FORCE_COMPILED_WASM
       ? "--wasm-execution=compiled"
@@ -110,6 +111,8 @@ export async function startMadaraDevNode(
   let runningNode: ChildProcess = null;
   process.once("exit", onProcessExit);
   process.once("SIGINT", onProcessInterrupt);
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  require("child_process").execSync(cmd + " setup");
   runningNode = spawn(cmd, args);
 
   runningNode.once("exit", () => {