From 1ea09139cc462df04d38a672aca3b41266016b80 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Wed, 22 Sep 2021 15:50:18 +0200 Subject: [PATCH] Add ci build (#30) --- .github/workflows/ci.yml | 380 +++++++++++++++++++++++++++++++++++++++ scripts/github/lib.rb | 12 ++ 2 files changed, 392 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 scripts/github/lib.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3d9ad09 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,380 @@ +name: CI + +on: + push: + branches: +# - master + - new_ci + tags: + - '[0-9]+.[0-9]+.[0-9]+' + pull_request: + branches: [master] + workflow_dispatch: + inputs: + verbose: + description: "Set --verbose to get verbose build output" + required: false + default: 'true' + +env: + VERBOSE: ${{ github.events.input.verbose }} + SUBWASM_VERSION: 0.12.0 + + +jobs: + + # this job set up dynamic configuration shared among jobs + config: + name: Set up Config + runs-on: ubuntu-latest + env: + ENABLE_CHECK: true + outputs: + check: ${{ steps.check.outputs.check }} + runs: ${{ steps.runs.outputs.runs }} + steps: + - id: check + run: echo "::set-output name=check::${{ env.ENABLE_CHECK }}" + + cancel_previous_runs: + name: Cancel Previous Runs + runs-on: ubuntu-20.04 + steps: + - uses: styfle/cancel-workflow-action@0.4.1 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + + build_primary_binaries: + name: Primary build ${{ matrix.binary }} for ${{ matrix.rust-target }} on (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + rust: [stable] + binary: [release] + env: + RUST_BACKTRACE: full + RUSTV: ${{ matrix.rust }} + TARGET: ${{ matrix.rust-target }} + RUST_BIN_DIR: target/${{ matrix.rust-target }}/debug + RELEASE_NAME: debug + steps: + - uses: actions/checkout@v2 + + - name: Print env + run: | + echo "RUST_BIN_DIR = ${{ env.RUST_BIN_DIR }} " + + # With rustup's nice new toml format, we just need to run rustup show to install the toolchain + # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 + - name: Setup Rust toolchain + run: rustup show + + - name: Install llvm-10 + run: sudo apt-get install llvm-10 clang-10 + + - uses: Swatinem/rust-cache@v1 + with: + key: ${{ matrix.os }}-${{ env.RUST_BIN_DIR }}-${{ matrix.binary }} + + - name: Release + if: ${{ matrix.binary == 'release' }} + run: | + cargo clean -p integritee-collator + cargo build --release + + # Upload artifacts + - name: Upload integritee-node + uses: actions/upload-artifact@v2 + with: + name: integritee-node-${{ github.sha }} + path: target/release/integritee-node + + - name: Slack Notification + uses: 8398a7/action-slack@v3 + if: failure() + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + + # Run all checks and unit test. This always run on debug mode + check: + name: Rust check ${{ matrix.check }} (${{ matrix.rust-target }}) + needs: config + if: ${{ needs.config.outputs.check == 'true' }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + rust: [stable] + rust-target: [x86_64-unknown-linux-gnu] + check: [fmt, clippy] + env: + RUST_BACKTRACE: full + RUSTV: ${{ matrix.rust }} + TARGET: ${{ matrix.rust-target }} + steps: + - uses: actions/checkout@v2 + + # With rustup's nice new toml format, we just need to run rustup show to install the toolchain + # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 + # - name: Setup Rust toolchain + # run: rustup show + + # - name: Install llvm-10 + # run: sudo apt-get install llvm-10 clang-10 + + # - uses: Swatinem/rust-cache@v1 + # with: + # key: ${{ matrix.rust-target }}-${{ matrix.check }} + + # - name: Fmt + # if: ${{ matrix.check == 'fmt' }} + # run: cargo fmt --all -- --check + + # - name: Clippy + # if: ${{ matrix.check == 'clippy' }} + # run: cargo clippy -- -D warnings + + build-runtimes: + name: Build Runtimes + runs-on: ubuntu-latest + strategy: + matrix: + runtime: ["integritee-node"] + steps: + - uses: actions/checkout@v2 + + - name: Cache target dir + uses: actions/cache@v2 + with: + path: "${{ github.workspace }}/runtime/target" + key: srtool-target-${{ matrix.runtime }}-${{ github.sha }} + restore-keys: | + srtool-target-${{ matrix.runtime }}- + srtool-target- + + - name: Srtool build + id: srtool_build + uses: chevdor/srtool-actions@v0.3.0 + with: + chain: ${{ matrix.runtime }} + runtime_dir: runtime + + - name: Summary + run: | + echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.runtime }}-srtool-digest.json + cat ${{ matrix.runtime }}-srtool-digest.json + echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}" + echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}" + + # We now get extra information thanks to subwasm + - name: Install subwasm + run: | + wget https://github.com/chevdor/subwasm/releases/download/v${{ env.SUBWASM_VERSION }}/subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb + sudo dpkg -i subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb + subwasm --version + + - name: Show Runtime information + shell: bash + run: | + subwasm info ${{ steps.srtool_build.outputs.wasm }} + subwasm info ${{ steps.srtool_build.outputs.wasm_compressed }} + subwasm --json info ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.runtime }}-info.json + subwasm --json info ${{ steps.srtool_build.outputs.wasm_compressed }} > ${{ matrix.runtime }}-compressed-info.json + + - name: Extract the metadata + shell: bash + run: | + subwasm meta ${{ steps.srtool_build.outputs.wasm }} + subwasm --json meta ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.runtime }}-metadata.json + +# This is unsupported it wants to diff the metadata with a running chain. i.e. wss://kusama--rpc.parity.io +# - name: Check the metadata diff +# shell: bash +# run: | +# subwasm diff ${{ steps.srtool_build.outputs.wasm }} --chain-b ${{ matrix.chain }} > ${{ matrix.chain }}-diff.txt +# cat ${{ matrix.chain }}-diff.txt + + - name: Upload ${{ matrix.runtime }} srtool json + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.runtime }}-srtool-json-${{ github.sha }} + path: | + ${{ matrix.runtime }}-srtool-digest.json + ${{ matrix.runtime }}-info.json + ${{ matrix.runtime }}-compressed-info.json + ${{ matrix.runtime }}-metadata.json +# ${{ matrix.runtime }}-diff.txt + + + - name: Upload ${{ matrix.runtime }} runtime + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.runtime }}-runtime-${{ github.sha }} + path: | + ${{ steps.srtool_build.outputs.wasm }} + ${{ steps.srtool_build.outputs.wasm_compressed }} + + - name: Slack Notification + uses: 8398a7/action-slack@v3 + if: failure() + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + create_artifacts: + needs: [ build_primary_binaries, build-runtimes ] + runs-on: ubuntu-20.04 + env: + CHAIN_SPEC: ${{ matrix.chain }}-${{ matrix.config }} + strategy: + fail-fast: false + matrix: + chain: [shell] + config: [kusama] + steps: + - name: Checkout + uses: actions/checkout@v2 + + # - uses: actions/download-artifact@v2 + # with: + # name: integritee-collator-${{ github.sha }} + + # - name: 'Create binaries for artifacts' + # # Note: there is no way to pass the parachain-id to the `build-spec` cmd, which which imposes to set + # # `DEFAULT_PARA_ID=2015` in the `polkadot-parachains/command`. + # run: | + # chmod +x ./integritee-collator + # ./integritee-collator export-genesis-state --chain ${{ env.CHAIN_SPEC }} --parachain-id 2015 > ${{ env.CHAIN_SPEC }}.state + # ./integritee-collator build-spec --chain ${{ env.CHAIN_SPEC }} --disable-default-bootnode --raw > ${{ env.CHAIN_SPEC }}.json + + + # - name: Compute file metadata + # id: vars + # run: | + # sha256sum ${{ env.CHAIN_SPEC }}.state >> checksums.txt + # sha256sum ${{ env.CHAIN_SPEC }}.json >> checksums.txt + + # - name: Upload ${{ env.CHAIN_SPEC }} Files + # uses: actions/upload-artifact@v2 + # with: + # name: ${{ env.CHAIN_SPEC }}-${{ github.sha }} + # path: | + # checksums.txt + # ${{ env.CHAIN_SPEC }}.state + # ${{ env.CHAIN_SPEC }}.json + + - name: Slack Notification + uses: 8398a7/action-slack@v3 + if: failure() + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + release: + name: Draft Release + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + needs: [create_artifacts, build_primary_binaries, check] + outputs: + release_url: ${{ steps.create-release.outputs.html_url }} + asset_upload_url: ${{ steps.create-release.outputs.upload_url }} + steps: + - uses: actions/checkout@v2 + + - name: Download Integritee Collator + uses: actions/download-artifact@v2 + with: + name: integritee-collator-${{ github.sha }} + + - name: Create required package.json + run: test -f package.json || echo '{}' >package.json + + - name: Changelog + uses: scottbrenner/generate-changelog-action@master + id: Changelog + + - name: Display structure of downloaded files + run: ls -R + working-directory: . + + - name: Release + id: create-release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + body: | + ${{ steps.Changelog.outputs.changelog }} + draft: true + files: | + integritee-node + + - name: Slack Notification + uses: 8398a7/action-slack@v3 + if: failure() + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + publish-runtimes: + name: Publish Runtimes + runs-on: ubuntu-latest + needs: ["release"] + strategy: + matrix: + runtime: ["integritee-node"] + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + + - name: Get runtime version + id: get-runtime-ver + run: | + ls + ls "${{ matrix.runtime }}-runtime-${{ github.sha }}" + runtime_ver="$(ruby -e 'require "./scripts/github/lib.rb"; puts get_runtime()')" + echo "::set-output name=runtime_ver::$runtime_ver" + + - name: Upload compact ${{ matrix.runtime }} wasm + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.release.outputs.asset_upload_url }} + asset_path: "${{ matrix.runtime }}-runtime-${{ github.sha }}/${{ matrix.runtime }}_runtime.compact.wasm" + asset_name: ${{ matrix.runtime }}_runtime-v${{ steps.get-runtime-ver.outputs.runtime_ver }}.compact.wasm + asset_content_type: application/wasm + + - name: Upload compressed ${{ matrix.runtime }} wasm + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.release.outputs.asset_upload_url }} + asset_path: "${{ matrix.runtime }}-runtime-${{ github.sha }}/${{ matrix.runtime }}_runtime.compact.compressed.wasm" + asset_name: ${{ matrix.runtime }}_runtime-v${{ steps.get-runtime-ver.outputs.runtime_ver }}.compact.compressed.wasm + asset_content_type: application/wasm + + - name: Slack Notification + uses: 8398a7/action-slack@v3 + if: failure() + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message) + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/scripts/github/lib.rb b/scripts/github/lib.rb new file mode 100644 index 0000000..f34f2c0 --- /dev/null +++ b/scripts/github/lib.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# A collection of helper functions that might be useful for various scripts + +# Gets the runtime version for a given runtime. +# Optionally accepts a path that is the root of the project which defaults to +# the current working directory +def get_runtime(path = '.') + File.open(path + "/runtime/src/lib.rs") do |f| + f.find { |l| l =~ /spec_version:/ }.match(/[0-9]+/)[0] + end +end