Allowing different chain binaries for integration tests

.changelog/unreleased/improvements/ibc-relayer/2003-different-chains-ci.md

@@ -0,0 +1,2 @@
+- Added the possibility to specify multiple chains for integration tests
+  ([#2003](https://github.com/informalsystems/ibc-rs/issues/2003))

.github/workflows/integration.yaml

@@ -104,8 +104,8 @@ jobs:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
-          CHAIN_COMMAND_PATH: ${{ matrix.chain.command }}
-          ACCOUNT_PREFIX: ${{ matrix.chain.account_prefix }}
+          CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
+          ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         with:
           max_attempts: 3
           timeout_minutes: 45
@@ -114,6 +114,117 @@ jobs:
               test -p ibc-integration-test --no-fail-fast -- \
               --nocapture --test-threads=2
 
+  multi-chains-test:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master'
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        first-package:
+          - package: gaia6
+            chain_command_path: gaiad
+            account_prefix: cosmos
+          - package: gaia7
+            chain_command_path: gaiad
+            account_prefix: cosmos
+          - package: gaia8
+            chain_command_path: gaiad
+            account_prefix: cosmos
+          - package: ibc-go-v2-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v3-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v4-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v5-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v6-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: wasmd
+            chain_command_path: wasmd
+            account_prefix: wasm
+          - package: evmos
+            chain_command_path: evmosd
+            account_prefix: evmos
+          - package: osmosis
+            chain_command_path: osmosisd
+            account_prefix: osmo
+        second-package:
+          - package: gaia6
+            chain_command_path: gaiad
+            account_prefix: cosmos
+          - package: gaia7
+            chain_command_path: gaiad
+            account_prefix: cosmos
+          - package: gaia8
+            chain_command_path: gaiad
+            account_prefix: cosmos
+          - package: ibc-go-v2-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v3-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v4-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v5-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: ibc-go-v6-simapp
+            chain_command_path: simd
+            account_prefix: cosmos
+          - package: wasmd
+            chain_command_path: wasmd
+            account_prefix: wasm
+          - package: evmos
+            chain_command_path: evmosd
+            account_prefix: evmos
+          - package: osmosis
+            chain_command_path: osmosisd
+            account_prefix: osmo
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: cachix/install-nix-action@v15
+        with:
+          install_url: https://nixos-nix-install-tests.cachix.org/serve/vij683ly7sl95nnhb67bdjjfabclr85m/install
+          install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+      - uses: cachix/cachix-action@v10
+        with:
+          name: cosmos
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: -p ibc-integration-test --no-fail-fast --no-run
+      - uses: nick-fields/retry@v2
+        env:
+          RUST_LOG: info
+          RUST_BACKTRACE: 1
+          NO_COLOR_LOG: 1
+          ACCOUNT_PREFIXES: ${{ matrix.first-package.account_prefix }},${{ matrix.second-package.account_prefix }}
+        with:
+          max_attempts: 3
+          timeout_minutes: 45
+          command: |
+            CHAIN_COMMAND_PATHS=$(nix shell .#${{ matrix.first-package.package }} -c which ${{ matrix.first-package.chain_command_path }}),$(nix shell .#${{ matrix.second-package.package }} -c which ${{ matrix.second-package.chain_command_path }}) \
+              nix shell .#python -c cargo \
+              test -p ibc-integration-test --no-fail-fast -- \
+              --nocapture --test-threads=2
+
   ordered-channel-test:
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -172,7 +283,7 @@ jobs:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
-          CHAIN_COMMAND_PATH: icad
+          CHAIN_COMMAND_PATHS: icad
         run: |
           nix shell .#python .#ica -c cargo \
             test -p ibc-integration-test --features ica --no-fail-fast -- \
@@ -214,8 +325,8 @@ jobs:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
-          CHAIN_COMMAND_PATH: ${{ matrix.chain.command }}
-          ACCOUNT_PREFIX: ${{ matrix.chain.account_prefix }}
+          CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
+          ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
           nix shell .#${{ matrix.chain.package }} -c cargo \
             test -p ibc-integration-test --features ics29-fee --no-fail-fast -- \
@@ -257,8 +368,8 @@ jobs:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
-          CHAIN_COMMAND_PATH: ${{ matrix.chain.command }}
-          ACCOUNT_PREFIX: ${{ matrix.chain.account_prefix }}
+          CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
+          ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
           nix shell .#${{ matrix.chain.package }} -c cargo \
             test -p ibc-integration-test --features forward-packet --no-fail-fast -- \

tools/test-framework/src/bootstrap/init.rs

@@ -37,11 +37,16 @@ pub fn init_test() -> Result<TestConfig, Error> {
         install_logger(!no_color_log);
     });
 
-    let chain_command_path = env::var("CHAIN_COMMAND_PATH").unwrap_or_else(|_| "gaiad".to_string());
+    let chain_command_path =
+        env::var("CHAIN_COMMAND_PATHS").unwrap_or_else(|_| "gaiad".to_string());
+
+    let chain_command_paths = parse_chain_command_paths(chain_command_path);
 
     let base_chain_store_dir = env::var("CHAIN_STORE_DIR").unwrap_or_else(|_| "data".to_string());
 
-    let account_prefix = env::var("ACCOUNT_PREFIX").unwrap_or_else(|_| "cosmos".to_string());
+    let account_prefix = env::var("ACCOUNT_PREFIXES").unwrap_or_else(|_| "cosmos".to_string());
+
+    let account_prefixes = parse_chain_command_paths(account_prefix);
 
     let chain_store_dir = format!("{}/test-{}", base_chain_store_dir, random_u32());
 
@@ -55,14 +60,22 @@ pub fn init_test() -> Result<TestConfig, Error> {
         .unwrap_or(false);
 
     Ok(TestConfig {
-        chain_command_path,
+        chain_command_paths,
         chain_store_dir,
-        account_prefix,
+        account_prefixes,
         hang_on_fail,
         bootstrap_with_random_ids: false,
     })
 }
 
+fn parse_chain_command_paths(chain_command_path: String) -> Vec<String> {
+    let patterns: Vec<String> = chain_command_path
+        .split(',')
+        .map(|chain_binary| chain_binary.to_string())
+        .collect();
+    patterns
+}
+
 /**
    Install the [`tracing_subscriber`] logger handlers so that logs will
    be displayed during test.

tools/test-framework/src/bootstrap/single.rs

@@ -42,6 +42,7 @@ pub fn bootstrap_single_node(
     use_random_id: bool,
     config_modifier: impl FnOnce(&mut toml::Value) -> Result<(), Error>,
     genesis_modifier: impl FnOnce(&mut serde_json::Value) -> Result<(), Error>,
+    chain_number: usize,
 ) -> Result<FullNode, Error> {
     let stake_denom = Denom::base("stake");
 
@@ -59,7 +60,7 @@ pub fn bootstrap_single_node(
     let initial_stake = Token::new(stake_denom, initial_amount);
     let initial_coin = Token::new(denom.clone(), initial_amount);
 
-    let chain_driver = builder.new_chain(prefix, use_random_id)?;
+    let chain_driver = builder.new_chain(prefix, use_random_id, chain_number)?;
 
     chain_driver.initialize()?;
 

tools/test-framework/src/chain/builder.rs

@@ -30,14 +30,14 @@ pub struct ChainBuilder {
        may return [`ChainDriver`]s bound to different chain commands
        for testing with multiple chain implementations.
     */
-    pub command_path: String,
+    pub command_paths: Vec<String>,
 
     /**
        The filesystem path to store the data files used by the chain.
     */
     pub base_store_dir: String,
 
-    pub account_prefix: String,
+    pub account_prefixes: Vec<String>,
 
     pub runtime: Arc<Runtime>,
 }
@@ -47,15 +47,15 @@ impl ChainBuilder {
        Create a new `ChainBuilder`.
     */
     pub fn new(
-        command_path: &str,
+        command_paths: Vec<String>,
         base_store_dir: &str,
-        account_prefix: &str,
+        account_prefixes: Vec<String>,
         runtime: Arc<Runtime>,
     ) -> Self {
         Self {
-            command_path: command_path.to_string(),
+            command_paths,
             base_store_dir: base_store_dir.to_string(),
-            account_prefix: account_prefix.to_string(),
+            account_prefixes,
             runtime,
         }
     }
@@ -65,9 +65,9 @@ impl ChainBuilder {
     */
     pub fn new_with_config(config: &TestConfig, runtime: Arc<Runtime>) -> Self {
         Self::new(
-            &config.chain_command_path,
+            config.chain_command_paths.clone(),
             &format!("{}", config.chain_store_dir.display()),
-            &config.account_prefix,
+            config.account_prefixes.clone(),
             runtime,
         )
     }
@@ -86,8 +86,18 @@ impl ChainBuilder {
        a [`ChainDriver`] configured with a chain ID  like
        `"ibc-alpha-f5a2a988"`.
     */
-    pub fn new_chain(&self, prefix: &str, use_random_id: bool) -> Result<ChainDriver, Error> {
-        let chain_type = ChainType::from_str(&self.command_path[..])?;
+    pub fn new_chain(
+        &self,
+        prefix: &str,
+        use_random_id: bool,
+        chain_number: usize,
+    ) -> Result<ChainDriver, Error> {
+        // If there are more spawned chains than given chain binaries, take the N-th position modulo
+        // the number of chain binaries given. Same for account prefix.
+        let chain_number = chain_number % self.command_paths.len();
+        let account_number = chain_number % self.account_prefixes.len();
+
+        let chain_type = ChainType::from_str(&self.command_paths[chain_number])?;
 
         let chain_id = chain_type.chain_id(prefix, use_random_id);
 
@@ -100,10 +110,10 @@ impl ChainBuilder {
 
         let driver = ChainDriver::create(
             chain_type,
-            self.command_path.clone(),
+            self.command_paths[chain_number].clone(),
             chain_id,
             home_path,
-            self.account_prefix.clone(),
+            self.account_prefixes[account_number].clone(),
             rpc_port,
             grpc_port,
             grpc_web_port,

tools/test-framework/src/framework/binary/node.rs

@@ -116,6 +116,7 @@ where
             config.bootstrap_with_random_ids,
             |config| self.test.get_overrides().modify_node_config(config),
             |genesis| self.test.get_overrides().modify_genesis_file(genesis),
+            0,
         )?;
 
         let node_b = bootstrap_single_node(
@@ -124,6 +125,7 @@ where
             config.bootstrap_with_random_ids,
             |config| self.test.get_overrides().modify_node_config(config),
             |genesis| self.test.get_overrides().modify_genesis_file(genesis),
+            1,
         )?;
 
         let _node_process_a = node_a.process.clone();
@@ -148,6 +150,7 @@ where
             config.bootstrap_with_random_ids,
             |config| self.test.get_overrides().modify_node_config(config),
             |genesis| self.test.get_overrides().modify_genesis_file(genesis),
+            0,
         )?;
 
         let _node_process = node.process.clone();

tools/test-framework/src/framework/nary/node.rs

@@ -62,6 +62,7 @@ where
                 config.bootstrap_with_random_ids,
                 |config| self.test.get_overrides().modify_node_config(config),
                 |genesis| self.test.get_overrides().modify_genesis_file(genesis),
+                i,
             )?;
 
             node_processes.push(node.process.clone());

tools/test-framework/src/types/config.rs

@@ -26,9 +26,9 @@ pub struct TestConfig {
        `["gaiad5"]` in `extra_chain_command_paths`, so that binary chain tests
        will use `gaiad5` for the second chain being spawned.
     */
-    pub chain_command_path: String,
+    pub chain_command_paths: Vec<String>,
 
-    pub account_prefix: String,
+    pub account_prefixes: Vec<String>,
 
     /**
        The directory path for storing the chain and relayer files.