From 41c45af0da0f5780c8df7c78ce78da1f30fb8e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20L=C3=B3pez?= <2642849+elopez@users.noreply.github.com> Date: Wed, 3 Feb 2021 11:20:43 -0300 Subject: [PATCH] Run echidna tests in parallel (#571) * semver integration for improve testing with different solc versions * fixes * more fixes * GitHub Actions: macOS releases: Drop extra libgmp renaming * GitHub Actions: drop workaround for actions/virtual-environments#1811 This has been fixed upstream by GitHub. * GitHub Actions: upgrade cache and setup-python actions to v2 * GitHub Actions: prettify Linux artifact Currently the tar.gz file has several folders inside from where the binary used to be in the runner, clean those up using -C. * Make tests static This helps in the case of CI, and allows to run the tests on a separate container without having to set up any library or Haskell dependencies. * GitHub Actions: implement parallel testing This allows for parallel execution of the echidna tests, which helps improve testing speed, especially once newer solc versions are included in the pipeline. * GitHub Actions: install the exact solc needed instead of all versions * GitHub Actions: re-enable MacOS CI Co-authored-by: ggrieco-tob Co-authored-by: Gustavo Grieco <31542053+ggrieco-tob@users.noreply.github.com> --- .github/scripts/install-solc.sh | 36 +++++++----- .github/workflows/ci.yml | 97 +++++++++++++++++++++++---------- package.yaml | 8 +++ 3 files changed, 100 insertions(+), 41 deletions(-) diff --git a/.github/scripts/install-solc.sh b/.github/scripts/install-solc.sh index 1f8b14d21..3df55026e 100755 --- a/.github/scripts/install-solc.sh +++ b/.github/scripts/install-solc.sh @@ -1,13 +1,9 @@ -#! /bin/bash +#!/bin/bash # Adapted from https://github.com/commercialhaskell/stack set -eux -if [ -f $HOME/.local/bin/solc-0.4.25 ] && [ -f $HOME/.local/bin/solc-0.5.7 ]; then - exit 0 -fi - mkdir -p $HOME/.local/bin; travis_retry() { @@ -16,15 +12,29 @@ travis_retry() { } fetch_solc_linux() { - rm -Rf solc-static-linux; - wget https://github.com/ethereum/solidity/releases/download/v0.4.25/solc-static-linux; - chmod +x solc-static-linux; - mv solc-static-linux $HOME/.local/bin/solc-0.4.25; - wget https://github.com/ethereum/solidity/releases/download/v0.5.7/solc-static-linux; - chmod +x solc-static-linux; - mv solc-static-linux $HOME/.local/bin/solc-0.5.7; + VER="$1" + if [ ! -f "$HOME/.local/bin/solc-$VER" ]; then + rm -Rf solc-static-linux + wget "https://github.com/ethereum/solidity/releases/download/v$VER/solc-static-linux" + chmod +x solc-static-linux + mv solc-static-linux "$HOME/.local/bin/solc-$VER" + echo "Downloaded solc $VER" + else + echo "Skipped solc $VER, already present" + fi +} + +fetch_all_solc_linux() { + fetch_solc_linux "0.4.25" + fetch_solc_linux "0.5.7" + fetch_solc_linux "0.6.12" + fetch_solc_linux "0.7.5" } if [ "$HOST_OS" = "Linux" ]; then - travis_retry fetch_solc_linux + if [ "${SOLC_VER:-}" == "" ]; then + travis_retry fetch_all_solc_linux + else + travis_retry fetch_solc_linux "$SOLC_VER" + fi fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b914db71..b3edd89ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,8 @@ on: - master jobs: - test: + build: + name: Build Echidna runs-on: ${{ matrix.os }} strategy: matrix: @@ -39,38 +40,28 @@ jobs: uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '3.6' - name: Cache Local - id: cache-local - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.local/ key: ${{ runner.os }}-local-v3 - name: Cache Stack - id: cache-stack - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.stack key: ${{ runner.os }}-stack-v3 - name: Cache Cabal - id: cache-cabal - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.cabal key: ${{ runner.os }}-cabal-v3 - - name: Build Binaries - run: | - .github/scripts/install-solc.sh - .github/scripts/install-crytic-compile.sh - env: - HOST_OS: ${{ runner.os }} - - name: Build Libraries run: | .github/scripts/install-libsecp256k1.sh @@ -86,27 +77,77 @@ jobs: run: | stack install --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib - - name: Test - if: runner.os == 'Linux' - run: | - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$HOME/.local/lib" - export PATH="${PATH}:$HOME/.local/bin" - - for VER in "0.4.25" "0.5.7" ; do - cp "$HOME/.local/bin/solc-$VER" "$HOME/.local/bin/solc" - stack test --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib - done - - name: Amend and compress binaries (macOS) if: runner.os == 'macOS' run: .github/scripts/build-macos-release.sh - name: Compress binary (Linux) if: runner.os == 'Linux' - run: GZIP=-9 tar -czf echidna-test.tar.gz $HOME/.local/bin/echidna-test + run: GZIP=-9 tar -czf echidna-test.tar.gz -C $HOME/.local/bin/ echidna-test - name: Upload artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: echidna-test-${{ runner.os }} path: echidna-test.tar.gz + + - name: Build and copy test suite + if: runner.os == 'Linux' + run: | + stack build --test --no-run-tests --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib + cp "$(find "$PWD" -name echidna-testsuite -type f)" . + + - name: Upload testsuite + if: runner.os == 'Linux' + uses: actions/upload-artifact@v2 + with: + name: echidna-testsuite + path: echidna-testsuite + + + test: + name: Test Echidna with solc ${{ matrix.solc }} + runs-on: ubuntu-latest + needs: build + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + solc: + - "0.4.25" + - "0.5.7" + experimental: [false] + include: + - solc: "0.6.12" + experimental: true + - solc: "0.7.5" + experimental: true + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: '3.6' + + - name: Install dependencies + run: | + .github/scripts/install-solc.sh + .github/scripts/install-crytic-compile.sh + env: + HOST_OS: ${{ runner.os }} + SOLC_VER: ${{ matrix.solc }} + + - name: Download testsuite + uses: actions/download-artifact@v2 + with: + name: echidna-testsuite + + - name: Test + run: | + export PATH="${PATH}:$HOME/.local/bin" + cp "$HOME/.local/bin/solc-${{ matrix.solc }}" "$HOME/.local/bin/solc" + chmod +x echidna-testsuite + ./echidna-testsuite diff --git a/package.yaml b/package.yaml index c036bba8a..f54684ad1 100644 --- a/package.yaml +++ b/package.yaml @@ -84,3 +84,11 @@ tests: - tasty - tasty-hunit - tasty-quickcheck + when: + - condition: os(linux) + ghc-options: + - -threaded + - -static + - -O2 + cc-options: -static + ld-options: -static -pthread