From 0f9b09efd5669f93de0208d6f8d2f10f45ce11bf Mon Sep 17 00:00:00 2001 From: Hussein Ait Lahcen Date: Wed, 26 Jan 2022 13:59:59 +0100 Subject: [PATCH 01/11] fixes #535 --- Cargo.lock | 117 ++++++++++++++++++++++++++++++++++- integration-tests/Cargo.toml | 21 ++++--- node/Cargo.toml | 2 +- 3 files changed, 129 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 565f0614909..fa28974e717 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7611,6 +7611,23 @@ dependencies = [ "xcm-executor", ] +[[package]] +name = "pallet-xcm-benchmarks" +version = "0.9.8" +source = "git+https://github.com/paritytech/polkadot?branch=release-v0.9.13#7d8f00b90cd6d87780123b3e08ca120cfb0c6e50" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log 0.4.14", + "parity-scale-codec", + "scale-info", + "sp-runtime", + "sp-std", + "xcm", + "xcm-executor", +] + [[package]] name = "parachain-info" version = "0.1.0" @@ -8944,12 +8961,15 @@ source = "git+https://github.com/paritytech/polkadot?branch=release-v0.9.13#7d8f dependencies = [ "beefy-primitives", "bitvec", + "frame-benchmarking", "frame-election-provider-support", "frame-executive", "frame-support", "frame-system", + "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", + "hex-literal", "log 0.4.14", "pallet-authority-discovery", "pallet-authorship", @@ -8970,9 +8990,11 @@ dependencies = [ "pallet-multisig", "pallet-nicks", "pallet-offences", + "pallet-offences-benchmarking", "pallet-proxy", "pallet-scheduler", "pallet-session", + "pallet-session-benchmarking", "pallet-staking", "pallet-staking-reward-curve", "pallet-timestamp", @@ -9112,6 +9134,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "futures 0.3.19", "hex-literal", + "kusama-runtime", "kvdb", "kvdb-rocksdb", "lru 0.7.2", @@ -9196,6 +9219,7 @@ dependencies = [ "substrate-prometheus-endpoint", "thiserror", "tracing", + "westend-runtime", ] [[package]] @@ -10072,10 +10096,12 @@ dependencies = [ "bp-runtime", "bp-wococo", "bridge-runtime-common", + "frame-benchmarking", "frame-executive", "frame-support", "frame-system", "frame-system-rpc-runtime-api", + "hex-literal", "log 0.4.14", "pallet-authority-discovery", "pallet-authorship", @@ -13463,7 +13489,7 @@ dependencies = [ "chrono", "lazy_static", "matchers", - "parking_lot 0.11.2", + "parking_lot 0.9.0", "regex", "serde", "serde_json", @@ -13643,8 +13669,8 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" dependencies = [ - "cfg-if 1.0.0", - "rand 0.8.4", + "cfg-if 0.1.10", + "rand 0.6.5", "static_assertions", ] @@ -14315,6 +14341,90 @@ dependencies = [ "cc", ] +[[package]] +name = "westend-runtime" +version = "0.9.13" +source = "git+https://github.com/paritytech/polkadot?branch=release-v0.9.13#7d8f00b90cd6d87780123b3e08ca120cfb0c6e50" +dependencies = [ + "beefy-primitives", + "bitvec", + "frame-benchmarking", + "frame-election-provider-support", + "frame-executive", + "frame-support", + "frame-system", + "frame-system-benchmarking", + "frame-system-rpc-runtime-api", + "frame-try-runtime", + "hex-literal", + "log 0.4.14", + "pallet-authority-discovery", + "pallet-authorship", + "pallet-babe", + "pallet-bags-list", + "pallet-balances", + "pallet-collective", + "pallet-democracy 4.0.0-dev", + "pallet-election-provider-multi-phase", + "pallet-elections-phragmen", + "pallet-grandpa", + "pallet-identity", + "pallet-im-online", + "pallet-indices", + "pallet-membership", + "pallet-mmr-primitives", + "pallet-multisig", + "pallet-nicks", + "pallet-offences", + "pallet-offences-benchmarking", + "pallet-proxy", + "pallet-recovery", + "pallet-scheduler", + "pallet-session", + "pallet-session-benchmarking", + "pallet-society", + "pallet-staking", + "pallet-staking-reward-curve", + "pallet-sudo", + "pallet-timestamp", + "pallet-transaction-payment", + "pallet-transaction-payment-rpc-runtime-api", + "pallet-treasury", + "pallet-utility", + "pallet-vesting 4.0.0-dev", + "pallet-xcm", + "pallet-xcm-benchmarks", + "parity-scale-codec", + "polkadot-parachain", + "polkadot-primitives", + "polkadot-runtime-common", + "polkadot-runtime-parachains", + "rustc-hex", + "scale-info", + "serde", + "serde_derive", + "smallvec 1.8.0", + "sp-api", + "sp-authority-discovery", + "sp-block-builder", + "sp-consensus-babe", + "sp-core", + "sp-inherents", + "sp-io", + "sp-npos-elections", + "sp-offchain", + "sp-runtime", + "sp-session", + "sp-staking", + "sp-std", + "sp-transaction-pool", + "sp-version", + "substrate-wasm-builder", + "xcm", + "xcm-builder", + "xcm-executor", +] + [[package]] name = "which" version = "4.2.2" @@ -14491,6 +14601,7 @@ name = "xcm-executor" version = "0.9.13" source = "git+https://github.com/paritytech/polkadot?branch=release-v0.9.13#7d8f00b90cd6d87780123b3e08ca120cfb0c6e50" dependencies = [ + "frame-benchmarking", "frame-support", "impl-trait-for-tuples", "log 0.4.14", diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 1ce356507ab..076c9751a3d 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -100,11 +100,11 @@ sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0 # added on top of runtime for emulation of network paste = "1.0.5" -polkadot-core-primitives = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13" } +polkadot-core-primitives = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13", default-features = false } xcm-emulator = { git = "https://github.com/dzmitry-lahoda-forks/xcm-simulator", branch = "dz/0.9.13", default-features = false } -kusama-runtime = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13" } -polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13" } -polkadot-runtime-parachains = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13" } +kusama-runtime = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13", default-features = false } +polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13", default-features = false } +polkadot-runtime-parachains = { git = "https://github.com/paritytech/polkadot", branch = "release-v0.9.13", default-features = false } picasso-runtime = { package = "dali-runtime", path = "../runtime/dali", default-features = false } [dev-dependencies] @@ -125,9 +125,9 @@ std = [ "sp-block-builder/std", "sp-transaction-pool/std", "sp-inherents/std", - "support/std", + "support/std", "executive/std", - "frame-system/std", + "frame-system/std", "utility/std", "authorship/std", "balances/std", @@ -163,7 +163,11 @@ std = [ "xcm-executor/std", "aura/std", "sp-consensus-aura/std", - "scale-info/std", + "scale-info/std", + "kusama-runtime/std", + "polkadot-core-primitives/std", + "polkadot-primitives/std", + "polkadot-runtime-parachains/std", ] runtime-benchmarks = [ @@ -189,6 +193,9 @@ runtime-benchmarks = [ "democracy/runtime-benchmarks", "utility/runtime-benchmarks", "kusama-runtime/runtime-benchmarks", + "picasso-runtime/runtime-benchmarks", + "polkadot-primitives/runtime-benchmarks", + "polkadot-runtime-parachains/runtime-benchmarks", ] develop = [ diff --git a/node/Cargo.toml b/node/Cargo.toml index 7ba04523bec..769aa9a1546 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -91,7 +91,7 @@ hex = "0.4.3" [features] default = [] -runtime-benchmarks = ["picasso-runtime/runtime-benchmarks", "composable-runtime/runtime-benchmarks", "dali-runtime/runtime-benchmarks"] +runtime-benchmarks = ["polkadot-service/runtime-benchmarks", "picasso-runtime/runtime-benchmarks", "composable-runtime/runtime-benchmarks", "dali-runtime/runtime-benchmarks"] ocw = [] dali = ["dali-runtime"] composable = ["composable-runtime"] From 9db49454151e1bc6e0ef2d837fb85562c80bac26 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 14:21:21 +0100 Subject: [PATCH 02/11] don't skip integration tests --- .github/workflows/check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2d821850d75..b3413e6a9b7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -63,7 +63,7 @@ jobs: # TODO: https://github.com/ComposableFi/composable/issues/535 /home/runner/.cargo/bin/cargo install -f cargo-llvm-cov /home/runner/.cargo/bin/rustup component add llvm-tools-preview --toolchain=nightly-2021-11-08 - SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo +nightly-2021-11-08 llvm-cov --workspace --locked --release --verbose --features=runtime-benchmarks --exclude=integration-tests --lcov --output-path lcov.info + SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo +nightly-2021-11-08 llvm-cov --workspace --locked --release --verbose --features=runtime-benchmarks --lcov --output-path lcov.info - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: @@ -93,7 +93,7 @@ jobs: - uses: actions/checkout@v2 - name: Run Test run: | - SKIP_WASM_BUILD=1 cargo test --workspace --locked --release --verbose --features=runtime-benchmarks --exclude=integration-tests + SKIP_WASM_BUILD=1 cargo test --workspace --locked --release --verbose --features=runtime-benchmarks linters: name: Linters From 9716e3ed75cafa193f57ba3b39dd3815c5c5e2d6 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 14:28:55 +0100 Subject: [PATCH 03/11] remove ref: main --- .github/workflows/simnode.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/simnode.yml b/.github/workflows/simnode.yml index 092c209b003..1680878373d 100644 --- a/.github/workflows/simnode.yml +++ b/.github/workflows/simnode.yml @@ -20,8 +20,6 @@ jobs: docker system prune --force --all --volumes - uses: actions/checkout@v2 - with: - ref: main - name: Check for runtime changes id: check_runtime From b70b25701759f85c2410ae755f85d5a583c5c505 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:17:16 +0100 Subject: [PATCH 04/11] get github ref from action --- .github/workflows/benchmark.yml | 2 ++ .github/workflows/release-drafter.yml | 4 ++++ .github/workflows/simnode.yml | 2 ++ .maintain/common/lib.sh | 2 -- scripts/polkadot-launch/composable.json | 6 ++++++ 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 24f4df8b84f..e43073bd6f5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -29,6 +29,7 @@ jobs: - name: Check for runtime changes env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} + GITHUB_REF_NAME: ${{ github.sha }} id: check_runtime if: github.event.label.name == 'needs-benchmarks' run: .maintain/check_runtime.sh @@ -36,6 +37,7 @@ jobs: - name: Benchmark Test env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} + GITHUB_REF_NAME: ${{ github.sha }} if: env.RUNTIME_CHECK == 1 id: run_benchmarks run: .maintain/run_benchmarks.sh diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index c99521a2cfa..0e710015535 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -34,6 +34,8 @@ jobs: - name: Runtime wasm builds id: runtime_release + env: + GITHUB_REF_NAME: ${{ github.sha }} run: | .maintain/runtime_release.sh @@ -87,6 +89,8 @@ jobs: asset_content_type: application/octet-stream - name: 🔨 Build Composable Binary + env: + GITHUB_REF_NAME: ${{ github.sha }} run: | .maintain/client_release.sh diff --git a/.github/workflows/simnode.yml b/.github/workflows/simnode.yml index 1680878373d..0c2403a3583 100644 --- a/.github/workflows/simnode.yml +++ b/.github/workflows/simnode.yml @@ -25,11 +25,13 @@ jobs: id: check_runtime env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} + GITHUB_REF_NAME: ${{ github.sha }} run: .maintain/check_runtime.sh - name: Run Simnode env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} + GITHUB_REF_NAME: ${{ github.sha }} if: env.RUNTIME_CHECK == 1 id: run_simnode run: .maintain/run_simnode.sh diff --git a/.maintain/common/lib.sh b/.maintain/common/lib.sh index e1858c20c7e..26bfbf006f9 100644 --- a/.maintain/common/lib.sh +++ b/.maintain/common/lib.sh @@ -1,7 +1,5 @@ #!/bin/bash -GITHUB_REF_NAME=$(git rev-parse --abbrev-ref HEAD) - get_latest_release() { curl --silent "https://api.github.com/repos/$1/releases/latest" | # Get latest release from GitHub api grep '"tag_name":' | # Get tag line diff --git a/scripts/polkadot-launch/composable.json b/scripts/polkadot-launch/composable.json index c1fdeccb56d..7c890721ba3 100644 --- a/scripts/polkadot-launch/composable.json +++ b/scripts/polkadot-launch/composable.json @@ -26,6 +26,12 @@ "wsPort": 9977, "basePath": "/tmp/polkadot-launch/relaychains/dave", "port": 30777 + }, + { + "name": "ferdie", + "wsPort": 9977, + "basePath": "/tmp/polkadot-launch/relaychains/ferdie", + "port": 30777 } ], "genesis": { From 1c882e893d711035d63e5f6f291b1b10888749db Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:21:13 +0100 Subject: [PATCH 05/11] try base_ref --- .github/workflows/benchmark.yml | 8 +- .github/workflows/release-drafter.yml | 168 +++++++++++++------------- .github/workflows/simnode.yml | 8 +- 3 files changed, 92 insertions(+), 92 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e43073bd6f5..f5b5c3e3509 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -7,7 +7,7 @@ on: - main jobs: - benchmark: + benchmark: name: Benchmark runs-on: - self-hosted @@ -29,15 +29,15 @@ jobs: - name: Check for runtime changes env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.sha }} + GITHUB_REF_NAME: ${{ github.base_ref }} id: check_runtime if: github.event.label.name == 'needs-benchmarks' run: .maintain/check_runtime.sh - + - name: Benchmark Test env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.sha }} + GITHUB_REF_NAME: ${{ github.base_ref }} if: env.RUNTIME_CHECK == 1 id: run_benchmarks run: .maintain/run_benchmarks.sh diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 0e710015535..6584b6bed77 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -5,102 +5,102 @@ on: - releases jobs: build-and-publish: - runs-on: - - self-hosted - - linux - - x64 - - sre - strategy: + runs-on: + - self-hosted + - linux + - x64 + - sre + strategy: fail-fast: true steps: - - name: Clean up - continue-on-error: true - run: | - sudo chown -R $USER:$USER $GITHUB_WORKSPACE - docker system prune --force --all --volumes + - name: Clean up + continue-on-error: true + run: | + sudo chown -R $USER:$USER $GITHUB_WORKSPACE + docker system prune --force --all --volumes - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable + - uses: actions/checkout@v2 + with: + fetch-depth: 0 - - name: Install srtool - run: | - cargo install --git https://github.com/chevdor/srtool-cli - cargo install --locked --git https://github.com/chevdor/subwasm --tag v0.16.1 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable - - name: Runtime wasm builds - id: runtime_release - env: - GITHUB_REF_NAME: ${{ github.sha }} - run: | - .maintain/runtime_release.sh + - name: Install srtool + run: | + cargo install --git https://github.com/chevdor/srtool-cli + cargo install --locked --git https://github.com/chevdor/subwasm --tag v0.16.1 - - uses: release-drafter/release-drafter@v5 - id: release_drafter + - name: Runtime wasm builds + id: runtime_release + env: + GITHUB_REF_NAME: ${{ github.base_ref }} + run: | + .maintain/runtime_release.sh - - name: Set Permission - continue-on-error: true - run: | - sudo su runner + - uses: release-drafter/release-drafter@v5 + id: release_drafter - - uses: actions/setup-node@v2 - with: - node-version: '16' + - name: Set Permission + continue-on-error: true + run: | + sudo su runner - - name: Add runtime metadata to release body - run: | - cd scripts/github && npm install - printf "\n%s" "${{ steps.release_drafter.outputs.body }}" >> $GITHUB_WORKSPACE/release.md - npm run update-release-body -- --id=${{ steps.release_drafter.outputs.id }} --repo=composableFi/composable --body="$GITHUB_WORKSPACE/release.md" - cd $GITHUB_WORKSPACE + - uses: actions/setup-node@v2 + with: + node-version: "16" - - name: Upload Dali wasm - id: upload-dali-wasm - if: env.dali_wasm == 1 - uses: actions/upload-release-asset@v1 - with: - upload_url: ${{ steps.release_drafter.outputs.upload_url }} - asset_path: ./runtime/dali/target/srtool/release/wbuild/dali-runtime/dali_runtime.compact.wasm - asset_name: dali_runtime.compact.wasm - asset_content_type: application/octet-stream + - name: Add runtime metadata to release body + run: | + cd scripts/github && npm install + printf "\n%s" "${{ steps.release_drafter.outputs.body }}" >> $GITHUB_WORKSPACE/release.md + npm run update-release-body -- --id=${{ steps.release_drafter.outputs.id }} --repo=composableFi/composable --body="$GITHUB_WORKSPACE/release.md" + cd $GITHUB_WORKSPACE - - name: Upload Picasso wasm - id: upload-picasso-wasm - if: env.picasso_wasm == 1 - uses: actions/upload-release-asset@v1 - with: - upload_url: ${{ steps.release_drafter.outputs.upload_url }} - asset_path: ./runtime/picasso/target/srtool/release/wbuild/picasso-runtime/picasso_runtime.compact.wasm - asset_name: picasso_runtime.compact.wasm - asset_content_type: application/octet-stream + - name: Upload Dali wasm + id: upload-dali-wasm + if: env.dali_wasm == 1 + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.release_drafter.outputs.upload_url }} + asset_path: ./runtime/dali/target/srtool/release/wbuild/dali-runtime/dali_runtime.compact.wasm + asset_name: dali_runtime.compact.wasm + asset_content_type: application/octet-stream - - name: Upload Composable wasm - id: upload-composable-wasm - if: env.composable_wasm == 1 - uses: actions/upload-release-asset@v1 - with: - upload_url: ${{ steps.release_drafter.outputs.upload_url }} - asset_path: ./runtime/composable/target/srtool/release/wbuild/composable-runtime/composable_runtime.compact.wasm - asset_name: composable_runtime.compact.wasm - asset_content_type: application/octet-stream + - name: Upload Picasso wasm + id: upload-picasso-wasm + if: env.picasso_wasm == 1 + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.release_drafter.outputs.upload_url }} + asset_path: ./runtime/picasso/target/srtool/release/wbuild/picasso-runtime/picasso_runtime.compact.wasm + asset_name: picasso_runtime.compact.wasm + asset_content_type: application/octet-stream - - name: 🔨 Build Composable Binary - env: - GITHUB_REF_NAME: ${{ github.sha }} - run: | - .maintain/client_release.sh + - name: Upload Composable wasm + id: upload-composable-wasm + if: env.composable_wasm == 1 + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.release_drafter.outputs.upload_url }} + asset_path: ./runtime/composable/target/srtool/release/wbuild/composable-runtime/composable_runtime.compact.wasm + asset_name: composable_runtime.compact.wasm + asset_content_type: application/octet-stream - - name: Upload Composable Node Binaries - id: upload-composable-binary - if: env.client_release == 1 - uses: actions/upload-release-asset@v1 + - name: 🔨 Build Composable Binary + env: + GITHUB_REF_NAME: ${{ github.base_ref }} + run: | + .maintain/client_release.sh - with: - upload_url: ${{ steps.release_drafter.outputs.upload_url }} - asset_path: ./target/release/composable - asset_name: composable - asset_content_type: application/octet-stream + - name: Upload Composable Node Binaries + id: upload-composable-binary + if: env.client_release == 1 + uses: actions/upload-release-asset@v1 + + with: + upload_url: ${{ steps.release_drafter.outputs.upload_url }} + asset_path: ./target/release/composable + asset_name: composable + asset_content_type: application/octet-stream diff --git a/.github/workflows/simnode.yml b/.github/workflows/simnode.yml index 0c2403a3583..7727c0d17ac 100644 --- a/.github/workflows/simnode.yml +++ b/.github/workflows/simnode.yml @@ -6,7 +6,7 @@ on: - main jobs: simnode: - runs-on: + runs-on: - self-hosted - linux - x64 @@ -18,20 +18,20 @@ jobs: run: | sudo chown -R $USER:$USER $GITHUB_WORKSPACE docker system prune --force --all --volumes - + - uses: actions/checkout@v2 - name: Check for runtime changes id: check_runtime env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.sha }} + GITHUB_REF_NAME: ${{ github.base_ref }} run: .maintain/check_runtime.sh - name: Run Simnode env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.sha }} + GITHUB_REF_NAME: ${{ github.base_ref }} if: env.RUNTIME_CHECK == 1 id: run_simnode run: .maintain/run_simnode.sh From 009c6919c089ffbbbe74a4aa3e9fa71698316639 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:28:14 +0100 Subject: [PATCH 06/11] try GITHUB_SHA --- .github/workflows/benchmark.yml | 6 ++-- .github/workflows/release-drafter.yml | 4 +-- .github/workflows/simnode.yml | 4 +-- .maintain/check_runtime.sh | 13 ++++---- .maintain/client_release.sh | 6 ++-- .maintain/common/lib.sh | 8 ++--- .maintain/run_benchmarks.sh | 40 ++++++++++++------------- .maintain/run_simnode.sh | 4 +-- .maintain/runtime_release.sh | 20 ++++++------- scripts/polkadot-launch/composable.json | 4 +-- 10 files changed, 51 insertions(+), 58 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f5b5c3e3509..7ece9b481d2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,13 +23,11 @@ jobs: docker system prune --force --all --volumes - uses: actions/checkout@v2 - with: - ref: main - name: Check for runtime changes env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.base_ref }} + id: check_runtime if: github.event.label.name == 'needs-benchmarks' run: .maintain/check_runtime.sh @@ -37,7 +35,7 @@ jobs: - name: Benchmark Test env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.base_ref }} + if: env.RUNTIME_CHECK == 1 id: run_benchmarks run: .maintain/run_benchmarks.sh diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 6584b6bed77..a04510dc215 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -35,7 +35,7 @@ jobs: - name: Runtime wasm builds id: runtime_release env: - GITHUB_REF_NAME: ${{ github.base_ref }} + run: | .maintain/runtime_release.sh @@ -90,7 +90,7 @@ jobs: - name: 🔨 Build Composable Binary env: - GITHUB_REF_NAME: ${{ github.base_ref }} + run: | .maintain/client_release.sh diff --git a/.github/workflows/simnode.yml b/.github/workflows/simnode.yml index 7727c0d17ac..4aae2ee3a42 100644 --- a/.github/workflows/simnode.yml +++ b/.github/workflows/simnode.yml @@ -25,13 +25,13 @@ jobs: id: check_runtime env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.base_ref }} + run: .maintain/check_runtime.sh - name: Run Simnode env: BASE_BRANCH: ${{ github.event.pull_request.base.ref }} - GITHUB_REF_NAME: ${{ github.base_ref }} + if: env.RUNTIME_CHECK == 1 id: run_simnode run: .maintain/run_simnode.sh diff --git a/.maintain/check_runtime.sh b/.maintain/check_runtime.sh index fbe276cf70b..057938b8f8e 100755 --- a/.maintain/check_runtime.sh +++ b/.maintain/check_runtime.sh @@ -14,18 +14,17 @@ VERSIONS_FILES=( "runtime/picasso/src/lib.rs,picasso,picasso" "runtime/dali/src/lib.rs,dali-chachacha,dali" - "runtime/composable/src/lib.rs,composable,composable" + "runtime/composable/src/lib.rs,composable,composable" ) - echo "make sure the main branch and release tag are available in shallow clones" - git fetch --depth="${GIT_DEPTH:-100}" origin "${BASE_BRANCH}" +echo "make sure the main branch and release tag are available in shallow clones" +git fetch --depth="${GIT_DEPTH:-100}" origin "${BASE_BRANCH}" simnode_check() { VERSIONS_FILE="$1" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$2" && check_runtime "$VERSIONS_FILE" "$2" - then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$2" && check_runtime "$VERSIONS_FILE" "$2"; then echo "Wasm sources have changed for $3" - echo "RUNTIME_CHECK=1" >> $GITHUB_ENV + echo "RUNTIME_CHECK=1" >>$GITHUB_ENV fi } @@ -33,7 +32,7 @@ for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do boldprint "Check if the wasm sources changed for $chain" simnode_check $output $folder $chain - done <<< "$i" + done <<<"$i" done # dropped through. there's something wrong; exit 1. diff --git a/.maintain/client_release.sh b/.maintain/client_release.sh index 85861f04fce..4c078fe2860 100755 --- a/.maintain/client_release.sh +++ b/.maintain/client_release.sh @@ -10,14 +10,12 @@ #shellcheck source=../common/lib.sh . "$(dirname "${0}")/./common/lib.sh" -RELEASE_VERSION=$(git tag --sort=committerdate | grep -E '^v[0-9]' | tail -1 ) +RELEASE_VERSION=$(git tag --sort=committerdate | grep -E '^v[0-9]' | tail -1) # Because this script runs when a tag has been published, the previous tag is the # last two tags PREV_TAG=$(gh release list -L=2 | sed -n '2 p' | awk '{print $(NF-1)}') - -if has_client_changes "${PREV_TAG}" "${GITHUB_REF_NAME}" -then +if has_client_changes "${PREV_TAG}" "${GITHUB_SHA}"; then boldprint "Building new client binaries" cargo build --release -p composable tar -czvf composable-"${RELEASE_VERSION}".tar.gz target/release/composable diff --git a/.maintain/common/lib.sh b/.maintain/common/lib.sh index 26bfbf006f9..50647404cc7 100644 --- a/.maintain/common/lib.sh +++ b/.maintain/common/lib.sh @@ -49,9 +49,9 @@ has_client_changes() { # checks if the spec/impl version has increased check_runtime() { VERSIONS_FILE="$1" - add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")" - sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")" if [ "${add_spec_version}" != "${sub_spec_version}" ]; then @@ -68,9 +68,9 @@ check_runtime() { # check for impl_version updates: if only the impl versions changed, we assume # there is no consensus-critical logic that has changed. - add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')" - sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')" # see if the impl version changed diff --git a/.maintain/run_benchmarks.sh b/.maintain/run_benchmarks.sh index 9775dd3116d..2d7da88a42c 100755 --- a/.maintain/run_benchmarks.sh +++ b/.maintain/run_benchmarks.sh @@ -17,26 +17,26 @@ steps=50 repeat=20 pallets=( - oracle - frame_system - timestamp - session - balances - indices - membership - treasury - scheduler - collective - democracy - collator_selection - utility - lending - dutch_auction + oracle + frame_system + timestamp + session + balances + indices + membership + treasury + scheduler + collective + democracy + collator_selection + utility + lending + dutch_auction ) /home/runner/.cargo/bin/rustup install nightly -/home/runner/.cargo/bin/rustup target add wasm32-unknown-unknown --toolchain nightly -/home/runner/.cargo/bin/cargo build --release -p composable --features=runtime-benchmarks +/home/runner/.cargo/bin/rustup target add wasm32-unknown-unknown --toolchain nightly +/home/runner/.cargo/bin/cargo build --release -p composable --features=runtime-benchmarks run_benchmarks() { OUTPUT=$1 @@ -62,16 +62,16 @@ run_benchmarks() { USERNAME=$(gcloud secrets versions access latest --secret=github-api-username) PASSWORD=$(gcloud secrets versions access latest --secret=github-api-token) git remote set-url origin https://"$USERNAME":"$PASSWORD"@github.com/ComposableFi/composable.git - git pull origin "$GITHUB_REF_NAME" + git pull origin "$GITHUB_SHA" git add runtime/"$FOLDER" git commit -m "Updates weights for $CHAIN" - git push origin "$GITHUB_REF_NAME" + git push origin "$GITHUB_SHA" # ToDO: Setup gpg signing and create a bot account for pushing } for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then run_benchmarks $output $chain $folder fi done <<<"$i" diff --git a/.maintain/run_simnode.sh b/.maintain/run_simnode.sh index d057b3a6aaa..21afbfa9bf1 100755 --- a/.maintain/run_simnode.sh +++ b/.maintain/run_simnode.sh @@ -18,7 +18,7 @@ VERSIONS_FILES=( /home/runner/.cargo/bin/rustup target add wasm32-unknown-unknown --toolchain nightly /home/runner/.cargo/bin/cargo build --release -p simnode sudo chown -R runner:runner target/release/simnode && sudo chmod +x target/release/simnode -sudo chown -R runner:runner /tmp/db +sudo chown -R runner:runner /tmp/db YDATE=$(date -d yesterday +'%m-%d-%Y') run_simnode() { @@ -35,7 +35,7 @@ run_simnode() { for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then # shellcheck disable=SC2086 run_simnode $chain fi diff --git a/.maintain/runtime_release.sh b/.maintain/runtime_release.sh index 6bc358ad033..3123a81e046 100755 --- a/.maintain/runtime_release.sh +++ b/.maintain/runtime_release.sh @@ -13,8 +13,8 @@ # shellcheck disable=SC2039 VERSIONS_FILES=( "runtime/picasso/src/lib.rs,picasso,picasso" - "runtime/dali/src/lib.rs,dali-chachacha,dali" - "runtime/composable/src/lib.rs,composable,composable" + "runtime/dali/src/lib.rs,dali-chachacha,dali" + "runtime/composable/src/lib.rs,composable,composable" ) # Because this script runs when a tag has been published, the previous tag is the # last two tags @@ -25,26 +25,24 @@ CURRENT_TAG=$(gh release list -L=1 | sed -n '1 p' | awk '{print $(NF-1)}') cargo install --git https://github.com/chevdor/srtool-cli cargo install --locked --git https://github.com/chevdor/subwasm --tag v0.16.1 - -build_runtime () { +build_runtime() { chain=$3 # srtool for reproducible builds srtool build --package "$chain"-runtime --profile release --runtime-dir runtime/"$chain" # subwasm for runtime metadata - echo "# $chain Runtime " >> release.md - subwasm info ./runtime/"$chain"/target/srtool/release/wbuild/"$chain"-runtime/"$chain"_runtime.compact.wasm >> release.md + echo "# $chain Runtime " >>release.md + subwasm info ./runtime/"$chain"/target/srtool/release/wbuild/"$chain"-runtime/"$chain"_runtime.compact.wasm >>release.md } # Check which runtimes have changed and build them for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${PREV_TAG}" "${GITHUB_REF_NAME}" "$folder" - then + if has_runtime_changes "${PREV_TAG}" "${GITHUB_SHA}" "$folder"; then build_runtime $output $chain $folder CHANGES=gh view release tag $CURRENT_TAG - echo $CHANGES | sed '1,/--/ d' >> release.md - echo "$chain-wasm=1" >> "$GITHUB_ENV" + echo $CHANGES | sed '1,/--/ d' >>release.md + echo "$chain-wasm=1" >>"$GITHUB_ENV" fi - done <<< "$i" + done <<<"$i" done diff --git a/scripts/polkadot-launch/composable.json b/scripts/polkadot-launch/composable.json index 7c890721ba3..98254046276 100644 --- a/scripts/polkadot-launch/composable.json +++ b/scripts/polkadot-launch/composable.json @@ -29,9 +29,9 @@ }, { "name": "ferdie", - "wsPort": 9977, + "wsPort": 9978, "basePath": "/tmp/polkadot-launch/relaychains/ferdie", - "port": 30777 + "port": 30778 } ], "genesis": { From a341bb7254e905f25fcb1e2c4a0cb159dba5d38f Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:30:59 +0100 Subject: [PATCH 07/11] GITHUB_REF_NAME --- .github/workflows/devnet-release.yml | 144 +++++++++++++-------------- .maintain/check_runtime.sh | 2 +- .maintain/client_release.sh | 2 +- .maintain/common/lib.sh | 8 +- .maintain/run_benchmarks.sh | 6 +- .maintain/run_simnode.sh | 2 +- .maintain/runtime_release.sh | 2 +- 7 files changed, 82 insertions(+), 84 deletions(-) diff --git a/.github/workflows/devnet-release.yml b/.github/workflows/devnet-release.yml index d0a444cc788..ae4ae5b43a6 100644 --- a/.github/workflows/devnet-release.yml +++ b/.github/workflows/devnet-release.yml @@ -13,76 +13,74 @@ jobs: group: deploy-devnet cancel-in-progress: false steps: - - name: Clean up - continue-on-error: true - run: | - sudo chown -R $USER:$USER $GITHUB_WORKSPACE - docker system prune --force --all --volumes - - - - uses: actions/checkout@v2 - - - uses: google-github-actions/setup-gcloud@master - with: - service_account_key: ${{ secrets.GCP_CREDENTIALS }} - export_default_credentials: true - - - uses: actions/setup-python@v2 - - - uses: cachix/install-nix-action@v16 - with: - nix_path: nixpkgs=channel:nixos-unstable - - - uses: cachix/cachix-action@v10 - with: - name: composable-community - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - extraPullNames: composable-community - - - name: Set env - run: | - echo "RELEASE_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - - name: Push artifact - run: | - /home/runner/.cargo/bin/cargo build --release - tar -czvf composable-picasso-${{ env.RELEASE_VERSION }}.tar.gz target/release/composable - gsutil mv *.tar.gz gs://composable-binaries/community-releases/picasso/ - - - name: Load state - run: | - cd nix - - echo $(cat devnet.json | jq --arg version "${{ env.RELEASE_VERSION }}" '.composable.version = $version' | jq --arg hash "$(nix-prefetch-url https://storage.googleapis.com/composable-binaries/community-releases/picasso/composable-picasso-${{ env.RELEASE_VERSION }}.tar.gz)" '.composable.hash = $hash') > devnet.json - - jq --null-input --arg client_email "$GCP_DEVNET_SERVICE_ACCOUNT" --arg project_id "$GCP_PROJECT_ID" --arg key "\"$GCP_DEVNET_SERVICE_ACCOUNT_KEY\"" '{ "project_id": $project_id, "private_key": ($key | fromjson), "client_email": $client_email }' > ops.json - - if gsutil -q stat $NIXOPS_STATE_URL/$NIXOPS_STATE; - then - gsutil cp $NIXOPS_STATE_URL/$NIXOPS_STATE $NIXOPS_STATE - else - nix develop .#deploy --impure --command nixops create -d devnet-gce - fi - env: - NIXOPS_STATE_URL: "gs://composable-state" - NIXOPS_STATE: "deployment.nixops" - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - GCP_DEVNET_SERVICE_ACCOUNT: ${{ secrets.GCP_DEVNET_SERVICE_ACCOUNT }} - GCP_DEVNET_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DEVNET_SERVICE_ACCOUNT_KEY }} - - - name: Deploy - run: | - cd nix - nix develop .#deploy --impure --command nixops deploy --check --confirm -d devnet-gce - env: - NIXOPS_STATE: "deployment.nixops" - - - name: Store state - if: always() - run: | - cd nix - gsutil cp $NIXOPS_STATE $NIXOPS_STATE_URL/ - env: - NIXOPS_STATE: "deployment.nixops" - NIXOPS_STATE_URL: "gs://composable-state" - + - name: Clean up + continue-on-error: true + run: | + sudo chown -R $USER:$USER $GITHUB_WORKSPACE + docker system prune --force --all --volumes + + - uses: actions/checkout@v2 + + - uses: google-github-actions/setup-gcloud@master + with: + service_account_key: ${{ secrets.GCP_CREDENTIALS }} + export_default_credentials: true + + - uses: actions/setup-python@v2 + + - uses: cachix/install-nix-action@v16 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - uses: cachix/cachix-action@v10 + with: + name: composable-community + authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" + extraPullNames: composable-community + + - name: Set env + run: | + echo "RELEASE_VERSION=$GITHUB_REF_NAME" >> $GITHUB_ENV + + - name: Push artifact + run: | + /home/runner/.cargo/bin/cargo build --release + tar -czvf composable-picasso-${{ env.RELEASE_VERSION }}.tar.gz target/release/composable + gsutil mv *.tar.gz gs://composable-binaries/community-releases/picasso/ + + - name: Load state + run: | + cd nix + + echo $(cat devnet.json | jq --arg version "${{ env.RELEASE_VERSION }}" '.composable.version = $version' | jq --arg hash "$(nix-prefetch-url https://storage.googleapis.com/composable-binaries/community-releases/picasso/composable-picasso-${{ env.RELEASE_VERSION }}.tar.gz)" '.composable.hash = $hash') > devnet.json + + jq --null-input --arg client_email "$GCP_DEVNET_SERVICE_ACCOUNT" --arg project_id "$GCP_PROJECT_ID" --arg key "\"$GCP_DEVNET_SERVICE_ACCOUNT_KEY\"" '{ "project_id": $project_id, "private_key": ($key | fromjson), "client_email": $client_email }' > ops.json + + if gsutil -q stat $NIXOPS_STATE_URL/$NIXOPS_STATE; + then + gsutil cp $NIXOPS_STATE_URL/$NIXOPS_STATE $NIXOPS_STATE + else + nix develop .#deploy --impure --command nixops create -d devnet-gce + fi + env: + NIXOPS_STATE_URL: "gs://composable-state" + NIXOPS_STATE: "deployment.nixops" + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + GCP_DEVNET_SERVICE_ACCOUNT: ${{ secrets.GCP_DEVNET_SERVICE_ACCOUNT }} + GCP_DEVNET_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DEVNET_SERVICE_ACCOUNT_KEY }} + + - name: Deploy + run: | + cd nix + nix develop .#deploy --impure --command nixops deploy --check --confirm -d devnet-gce + env: + NIXOPS_STATE: "deployment.nixops" + + - name: Store state + if: always() + run: | + cd nix + gsutil cp $NIXOPS_STATE $NIXOPS_STATE_URL/ + env: + NIXOPS_STATE: "deployment.nixops" + NIXOPS_STATE_URL: "gs://composable-state" diff --git a/.maintain/check_runtime.sh b/.maintain/check_runtime.sh index 057938b8f8e..87efbc466c0 100755 --- a/.maintain/check_runtime.sh +++ b/.maintain/check_runtime.sh @@ -22,7 +22,7 @@ git fetch --depth="${GIT_DEPTH:-100}" origin "${BASE_BRANCH}" simnode_check() { VERSIONS_FILE="$1" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$2" && check_runtime "$VERSIONS_FILE" "$2"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$2" && check_runtime "$VERSIONS_FILE" "$2"; then echo "Wasm sources have changed for $3" echo "RUNTIME_CHECK=1" >>$GITHUB_ENV fi diff --git a/.maintain/client_release.sh b/.maintain/client_release.sh index 4c078fe2860..74e21e822d3 100755 --- a/.maintain/client_release.sh +++ b/.maintain/client_release.sh @@ -15,7 +15,7 @@ RELEASE_VERSION=$(git tag --sort=committerdate | grep -E '^v[0-9]' | tail -1) # last two tags PREV_TAG=$(gh release list -L=2 | sed -n '2 p' | awk '{print $(NF-1)}') -if has_client_changes "${PREV_TAG}" "${GITHUB_SHA}"; then +if has_client_changes "${PREV_TAG}" "${GITHUB_REF_NAME}"; then boldprint "Building new client binaries" cargo build --release -p composable tar -czvf composable-"${RELEASE_VERSION}".tar.gz target/release/composable diff --git a/.maintain/common/lib.sh b/.maintain/common/lib.sh index 50647404cc7..26bfbf006f9 100644 --- a/.maintain/common/lib.sh +++ b/.maintain/common/lib.sh @@ -49,9 +49,9 @@ has_client_changes() { # checks if the spec/impl version has increased check_runtime() { VERSIONS_FILE="$1" - add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")" - sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")" if [ "${add_spec_version}" != "${sub_spec_version}" ]; then @@ -68,9 +68,9 @@ check_runtime() { # check for impl_version updates: if only the impl versions changed, we assume # there is no consensus-critical logic that has changed. - add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')" - sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')" # see if the impl version changed diff --git a/.maintain/run_benchmarks.sh b/.maintain/run_benchmarks.sh index 2d7da88a42c..3fe64f938e2 100755 --- a/.maintain/run_benchmarks.sh +++ b/.maintain/run_benchmarks.sh @@ -62,16 +62,16 @@ run_benchmarks() { USERNAME=$(gcloud secrets versions access latest --secret=github-api-username) PASSWORD=$(gcloud secrets versions access latest --secret=github-api-token) git remote set-url origin https://"$USERNAME":"$PASSWORD"@github.com/ComposableFi/composable.git - git pull origin "$GITHUB_SHA" + git pull origin "$GITHUB_REF_NAME" git add runtime/"$FOLDER" git commit -m "Updates weights for $CHAIN" - git push origin "$GITHUB_SHA" + git push origin "$GITHUB_REF_NAME" # ToDO: Setup gpg signing and create a bot account for pushing } for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then run_benchmarks $output $chain $folder fi done <<<"$i" diff --git a/.maintain/run_simnode.sh b/.maintain/run_simnode.sh index 21afbfa9bf1..5d93a55af24 100755 --- a/.maintain/run_simnode.sh +++ b/.maintain/run_simnode.sh @@ -35,7 +35,7 @@ run_simnode() { for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then # shellcheck disable=SC2086 run_simnode $chain fi diff --git a/.maintain/runtime_release.sh b/.maintain/runtime_release.sh index 3123a81e046..b082fcf0270 100755 --- a/.maintain/runtime_release.sh +++ b/.maintain/runtime_release.sh @@ -38,7 +38,7 @@ build_runtime() { for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${PREV_TAG}" "${GITHUB_SHA}" "$folder"; then + if has_runtime_changes "${PREV_TAG}" "${GITHUB_REF_NAME}" "$folder"; then build_runtime $output $chain $folder CHANGES=gh view release tag $CURRENT_TAG echo $CHANGES | sed '1,/--/ d' >>release.md From 31bfe596f7685695c8eb8f937c806304fde65fba Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:38:31 +0100 Subject: [PATCH 08/11] back to GITHUB_SHA --- .github/workflows/benchmark.yml | 2 ++ .github/workflows/devnet-release.yml | 2 +- .github/workflows/simnode.yml | 2 ++ .maintain/check_runtime.sh | 2 +- .maintain/client_release.sh | 2 +- .maintain/common/lib.sh | 8 ++++---- .maintain/run_benchmarks.sh | 6 +++--- .maintain/run_simnode.sh | 2 +- .maintain/runtime_release.sh | 2 +- 9 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7ece9b481d2..3ed0d9d8ee5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,6 +23,8 @@ jobs: docker system prune --force --all --volumes - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Check for runtime changes env: diff --git a/.github/workflows/devnet-release.yml b/.github/workflows/devnet-release.yml index ae4ae5b43a6..39dcd73d2bb 100644 --- a/.github/workflows/devnet-release.yml +++ b/.github/workflows/devnet-release.yml @@ -40,7 +40,7 @@ jobs: - name: Set env run: | - echo "RELEASE_VERSION=$GITHUB_REF_NAME" >> $GITHUB_ENV + echo "RELEASE_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Push artifact run: | diff --git a/.github/workflows/simnode.yml b/.github/workflows/simnode.yml index 4aae2ee3a42..115c953fe4c 100644 --- a/.github/workflows/simnode.yml +++ b/.github/workflows/simnode.yml @@ -20,6 +20,8 @@ jobs: docker system prune --force --all --volumes - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Check for runtime changes id: check_runtime diff --git a/.maintain/check_runtime.sh b/.maintain/check_runtime.sh index 87efbc466c0..057938b8f8e 100755 --- a/.maintain/check_runtime.sh +++ b/.maintain/check_runtime.sh @@ -22,7 +22,7 @@ git fetch --depth="${GIT_DEPTH:-100}" origin "${BASE_BRANCH}" simnode_check() { VERSIONS_FILE="$1" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$2" && check_runtime "$VERSIONS_FILE" "$2"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$2" && check_runtime "$VERSIONS_FILE" "$2"; then echo "Wasm sources have changed for $3" echo "RUNTIME_CHECK=1" >>$GITHUB_ENV fi diff --git a/.maintain/client_release.sh b/.maintain/client_release.sh index 74e21e822d3..4c078fe2860 100755 --- a/.maintain/client_release.sh +++ b/.maintain/client_release.sh @@ -15,7 +15,7 @@ RELEASE_VERSION=$(git tag --sort=committerdate | grep -E '^v[0-9]' | tail -1) # last two tags PREV_TAG=$(gh release list -L=2 | sed -n '2 p' | awk '{print $(NF-1)}') -if has_client_changes "${PREV_TAG}" "${GITHUB_REF_NAME}"; then +if has_client_changes "${PREV_TAG}" "${GITHUB_SHA}"; then boldprint "Building new client binaries" cargo build --release -p composable tar -czvf composable-"${RELEASE_VERSION}".tar.gz target/release/composable diff --git a/.maintain/common/lib.sh b/.maintain/common/lib.sh index 26bfbf006f9..50647404cc7 100644 --- a/.maintain/common/lib.sh +++ b/.maintain/common/lib.sh @@ -49,9 +49,9 @@ has_client_changes() { # checks if the spec/impl version has increased check_runtime() { VERSIONS_FILE="$1" - add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")" - sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")" if [ "${add_spec_version}" != "${sub_spec_version}" ]; then @@ -68,9 +68,9 @@ check_runtime() { # check for impl_version updates: if only the impl versions changed, we assume # there is no consensus-critical logic that has changed. - add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')" - sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | + sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')" # see if the impl version changed diff --git a/.maintain/run_benchmarks.sh b/.maintain/run_benchmarks.sh index 3fe64f938e2..2d7da88a42c 100755 --- a/.maintain/run_benchmarks.sh +++ b/.maintain/run_benchmarks.sh @@ -62,16 +62,16 @@ run_benchmarks() { USERNAME=$(gcloud secrets versions access latest --secret=github-api-username) PASSWORD=$(gcloud secrets versions access latest --secret=github-api-token) git remote set-url origin https://"$USERNAME":"$PASSWORD"@github.com/ComposableFi/composable.git - git pull origin "$GITHUB_REF_NAME" + git pull origin "$GITHUB_SHA" git add runtime/"$FOLDER" git commit -m "Updates weights for $CHAIN" - git push origin "$GITHUB_REF_NAME" + git push origin "$GITHUB_SHA" # ToDO: Setup gpg signing and create a bot account for pushing } for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then run_benchmarks $output $chain $folder fi done <<<"$i" diff --git a/.maintain/run_simnode.sh b/.maintain/run_simnode.sh index 5d93a55af24..21afbfa9bf1 100755 --- a/.maintain/run_simnode.sh +++ b/.maintain/run_simnode.sh @@ -35,7 +35,7 @@ run_simnode() { for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then # shellcheck disable=SC2086 run_simnode $chain fi diff --git a/.maintain/runtime_release.sh b/.maintain/runtime_release.sh index b082fcf0270..3123a81e046 100755 --- a/.maintain/runtime_release.sh +++ b/.maintain/runtime_release.sh @@ -38,7 +38,7 @@ build_runtime() { for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${PREV_TAG}" "${GITHUB_REF_NAME}" "$folder"; then + if has_runtime_changes "${PREV_TAG}" "${GITHUB_SHA}" "$folder"; then build_runtime $output $chain $folder CHANGES=gh view release tag $CURRENT_TAG echo $CHANGES | sed '1,/--/ d' >>release.md From 1691d860dd8c39b48062f80a5b90cacd673d12be Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:45:24 +0100 Subject: [PATCH 09/11] checkout pull request commit --- .github/workflows/benchmark.yml | 1 + .github/workflows/simnode.yml | 3 ++- .maintain/check_runtime.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3ed0d9d8ee5..9063c80527d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,6 +25,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} - name: Check for runtime changes env: diff --git a/.github/workflows/simnode.yml b/.github/workflows/simnode.yml index 115c953fe4c..d85e4846bd5 100644 --- a/.github/workflows/simnode.yml +++ b/.github/workflows/simnode.yml @@ -21,7 +21,8 @@ jobs: - uses: actions/checkout@v2 with: - fetch-depth: 0 + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} - name: Check for runtime changes id: check_runtime diff --git a/.maintain/check_runtime.sh b/.maintain/check_runtime.sh index 057938b8f8e..7248fbe1210 100755 --- a/.maintain/check_runtime.sh +++ b/.maintain/check_runtime.sh @@ -5,7 +5,7 @@ # "auto-fail" the PR if there isn't a change in the runtime/src/lib.rs file # that alters the version. -#set -e # fail on any error +set -e # fail on any error #shellcheck source=../common/lib.sh . "$(dirname "${0}")/./common/lib.sh" From 1cd67e749647ca38cc7a201dcde80d8f7ad401d7 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:49:30 +0100 Subject: [PATCH 10/11] renamee to release-pipeline --- .github/workflows/{release-drafter.yml => release-pipeline.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{release-drafter.yml => release-pipeline.yml} (99%) diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-pipeline.yml similarity index 99% rename from .github/workflows/release-drafter.yml rename to .github/workflows/release-pipeline.yml index a04510dc215..0880763e3d3 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-pipeline.yml @@ -1,4 +1,4 @@ -name: Release Drafter +name: Release Pipeline on: push: branches: From 2a6f67477719c562afbfc6aa6f3e35330c1de174 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Wed, 26 Jan 2022 16:53:09 +0100 Subject: [PATCH 11/11] back to --- .github/workflows/devnet-release.yml | 2 +- .maintain/check_runtime.sh | 2 +- .maintain/client_release.sh | 2 +- .maintain/common/lib.sh | 10 ++++++---- .maintain/run_benchmarks.sh | 6 +++--- .maintain/run_simnode.sh | 2 +- .maintain/runtime_release.sh | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/devnet-release.yml b/.github/workflows/devnet-release.yml index 39dcd73d2bb..ae4ae5b43a6 100644 --- a/.github/workflows/devnet-release.yml +++ b/.github/workflows/devnet-release.yml @@ -40,7 +40,7 @@ jobs: - name: Set env run: | - echo "RELEASE_VERSION=$GITHUB_SHA" >> $GITHUB_ENV + echo "RELEASE_VERSION=$GITHUB_REF_NAME" >> $GITHUB_ENV - name: Push artifact run: | diff --git a/.maintain/check_runtime.sh b/.maintain/check_runtime.sh index 7248fbe1210..2b904f3458a 100755 --- a/.maintain/check_runtime.sh +++ b/.maintain/check_runtime.sh @@ -22,7 +22,7 @@ git fetch --depth="${GIT_DEPTH:-100}" origin "${BASE_BRANCH}" simnode_check() { VERSIONS_FILE="$1" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$2" && check_runtime "$VERSIONS_FILE" "$2"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$2" && check_runtime "$VERSIONS_FILE" "$2"; then echo "Wasm sources have changed for $3" echo "RUNTIME_CHECK=1" >>$GITHUB_ENV fi diff --git a/.maintain/client_release.sh b/.maintain/client_release.sh index 4c078fe2860..74e21e822d3 100755 --- a/.maintain/client_release.sh +++ b/.maintain/client_release.sh @@ -15,7 +15,7 @@ RELEASE_VERSION=$(git tag --sort=committerdate | grep -E '^v[0-9]' | tail -1) # last two tags PREV_TAG=$(gh release list -L=2 | sed -n '2 p' | awk '{print $(NF-1)}') -if has_client_changes "${PREV_TAG}" "${GITHUB_SHA}"; then +if has_client_changes "${PREV_TAG}" "${GITHUB_REF_NAME}"; then boldprint "Building new client binaries" cargo build --release -p composable tar -czvf composable-"${RELEASE_VERSION}".tar.gz target/release/composable diff --git a/.maintain/common/lib.sh b/.maintain/common/lib.sh index 50647404cc7..e1858c20c7e 100644 --- a/.maintain/common/lib.sh +++ b/.maintain/common/lib.sh @@ -1,5 +1,7 @@ #!/bin/bash +GITHUB_REF_NAME=$(git rev-parse --abbrev-ref HEAD) + get_latest_release() { curl --silent "https://api.github.com/repos/$1/releases/latest" | # Get latest release from GitHub api grep '"tag_name":' | # Get tag line @@ -49,9 +51,9 @@ has_client_changes() { # checks if the spec/impl version has increased check_runtime() { VERSIONS_FILE="$1" - add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + add_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")" - sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + sub_spec_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")" if [ "${add_spec_version}" != "${sub_spec_version}" ]; then @@ -68,9 +70,9 @@ check_runtime() { # check for impl_version updates: if only the impl versions changed, we assume # there is no consensus-critical logic that has changed. - add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + add_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')" - sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_SHA}" -- "${VERSIONS_FILE}" | + sub_impl_version="$(git diff "${LATEST_TAG_NAME}" "${GITHUB_REF_NAME}" -- "${VERSIONS_FILE}" | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')" # see if the impl version changed diff --git a/.maintain/run_benchmarks.sh b/.maintain/run_benchmarks.sh index 2d7da88a42c..3fe64f938e2 100755 --- a/.maintain/run_benchmarks.sh +++ b/.maintain/run_benchmarks.sh @@ -62,16 +62,16 @@ run_benchmarks() { USERNAME=$(gcloud secrets versions access latest --secret=github-api-username) PASSWORD=$(gcloud secrets versions access latest --secret=github-api-token) git remote set-url origin https://"$USERNAME":"$PASSWORD"@github.com/ComposableFi/composable.git - git pull origin "$GITHUB_SHA" + git pull origin "$GITHUB_REF_NAME" git add runtime/"$FOLDER" git commit -m "Updates weights for $CHAIN" - git push origin "$GITHUB_SHA" + git push origin "$GITHUB_REF_NAME" # ToDO: Setup gpg signing and create a bot account for pushing } for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then run_benchmarks $output $chain $folder fi done <<<"$i" diff --git a/.maintain/run_simnode.sh b/.maintain/run_simnode.sh index 21afbfa9bf1..5d93a55af24 100755 --- a/.maintain/run_simnode.sh +++ b/.maintain/run_simnode.sh @@ -35,7 +35,7 @@ run_simnode() { for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_SHA}" "$folder"; then + if has_runtime_changes "${BASE_BRANCH}" "${GITHUB_REF_NAME}" "$folder"; then # shellcheck disable=SC2086 run_simnode $chain fi diff --git a/.maintain/runtime_release.sh b/.maintain/runtime_release.sh index 3123a81e046..b082fcf0270 100755 --- a/.maintain/runtime_release.sh +++ b/.maintain/runtime_release.sh @@ -38,7 +38,7 @@ build_runtime() { for i in "${VERSIONS_FILES[@]}"; do while IFS=',' read -r output chain folder; do echo "check if the wasm sources changed for $chain" - if has_runtime_changes "${PREV_TAG}" "${GITHUB_SHA}" "$folder"; then + if has_runtime_changes "${PREV_TAG}" "${GITHUB_REF_NAME}" "$folder"; then build_runtime $output $chain $folder CHANGES=gh view release tag $CURRENT_TAG echo $CHANGES | sed '1,/--/ d' >>release.md