Run echidna tests in parallel (#571)

* semver integration for improve testing with different solc versions * fixes * more fixes * GitHub Actions: macOS releases: Drop extra libgmp renaming * GitHub Actions: drop workaround for actions/runner-images#1811 This has been fixed upstream by GitHub. * GitHub Actions: upgrade cache and setup-python actions to v2 * GitHub Actions: prettify Linux artifact Currently the tar.gz file has several folders inside from where the binary used to be in the runner, clean those up using -C. * Make tests static This helps in the case of CI, and allows to run the tests on a separate container without having to set up any library or Haskell dependencies. * GitHub Actions: implement parallel testing This allows for parallel execution of the echidna tests, which helps improve testing speed, especially once newer solc versions are included in the pipeline. * GitHub Actions: install the exact solc needed instead of all versions * GitHub Actions: re-enable MacOS CI Co-authored-by: ggrieco-tob <gustavo.grieco@trailofbits.com> Co-authored-by: Gustavo Grieco <31542053+ggrieco-tob@users.noreply.github.com>
crytic · Feb 3, 2021 · 41c45af · 41c45af
1 parent f35b468
commit 41c45af
Show file tree

Hide file tree

Showing 3 changed files with 100 additions and 41 deletions.
diff --git a/.github/scripts/install-solc.sh b/.github/scripts/install-solc.sh
@@ -1,13 +1,9 @@
-#! /bin/bash
+#!/bin/bash
 
 # Adapted from https://github.com/commercialhaskell/stack
 
 set -eux
 
-if [ -f $HOME/.local/bin/solc-0.4.25 ] && [ -f $HOME/.local/bin/solc-0.5.7 ]; then
-  exit 0
-fi
-
 mkdir -p $HOME/.local/bin;
 
 travis_retry() {
@@ -16,15 +12,29 @@ travis_retry() {
 }
 
 fetch_solc_linux() {
-  rm -Rf solc-static-linux;
-  wget https://github.com/ethereum/solidity/releases/download/v0.4.25/solc-static-linux;
-  chmod +x solc-static-linux;
-  mv solc-static-linux $HOME/.local/bin/solc-0.4.25;
-  wget https://github.com/ethereum/solidity/releases/download/v0.5.7/solc-static-linux;
-  chmod +x solc-static-linux;
-  mv solc-static-linux $HOME/.local/bin/solc-0.5.7;
+  VER="$1"
+  if [ ! -f "$HOME/.local/bin/solc-$VER" ]; then
+    rm -Rf solc-static-linux
+    wget "https://github.com/ethereum/solidity/releases/download/v$VER/solc-static-linux"
+    chmod +x solc-static-linux
+    mv solc-static-linux "$HOME/.local/bin/solc-$VER"
+    echo "Downloaded solc $VER"
+  else
+    echo "Skipped solc $VER, already present"
+  fi
+}
+
+fetch_all_solc_linux() {
+  fetch_solc_linux "0.4.25"
+  fetch_solc_linux "0.5.7"
+  fetch_solc_linux "0.6.12"
+  fetch_solc_linux "0.7.5"
 }
 
 if [ "$HOST_OS" = "Linux" ]; then
-  travis_retry fetch_solc_linux
+  if [ "${SOLC_VER:-}" == "" ]; then
+    travis_retry fetch_all_solc_linux
+  else
+    travis_retry fetch_solc_linux "$SOLC_VER"
+  fi
 fi
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,7 +9,8 @@ on:
     - master
 
 jobs:
-  test:
+  build:
+    name: Build Echidna
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -39,38 +40,28 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: '3.6'
 
       - name: Cache Local
-        id: cache-local
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ~/.local/
           key: ${{ runner.os }}-local-v3
 
       - name: Cache Stack
-        id: cache-stack
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ~/.stack
           key: ${{ runner.os }}-stack-v3
 
       - name: Cache Cabal
-        id: cache-cabal
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ~/.cabal
           key: ${{ runner.os }}-cabal-v3
 
-      - name: Build Binaries
-        run: |
-          .github/scripts/install-solc.sh
-          .github/scripts/install-crytic-compile.sh
-        env:
-          HOST_OS: ${{ runner.os }}
-
       - name: Build Libraries
         run: |
           .github/scripts/install-libsecp256k1.sh
@@ -86,27 +77,77 @@ jobs:
         run: |
           stack install --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib
 
-      - name: Test
-        if: runner.os == 'Linux'
-        run: |
-          export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$HOME/.local/lib"
-          export PATH="${PATH}:$HOME/.local/bin"
-
-          for VER in "0.4.25" "0.5.7" ; do
-            cp "$HOME/.local/bin/solc-$VER" "$HOME/.local/bin/solc"
-            stack test --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib
-          done
-
       - name: Amend and compress binaries (macOS)
         if: runner.os == 'macOS'
         run: .github/scripts/build-macos-release.sh
 
       - name: Compress binary (Linux)
         if: runner.os == 'Linux'
-        run: GZIP=-9 tar -czf echidna-test.tar.gz $HOME/.local/bin/echidna-test
+        run: GZIP=-9 tar -czf echidna-test.tar.gz -C $HOME/.local/bin/ echidna-test
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v2
         with:
           name: echidna-test-${{ runner.os }}
           path: echidna-test.tar.gz
+
+      - name: Build and copy test suite
+        if: runner.os == 'Linux'
+        run: |
+          stack build --test --no-run-tests --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib
+          cp "$(find "$PWD" -name echidna-testsuite -type f)" .
+
+      - name: Upload testsuite
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v2
+        with:
+          name: echidna-testsuite
+          path: echidna-testsuite
+
+
+  test:
+    name: Test Echidna with solc ${{ matrix.solc }}
+    runs-on: ubuntu-latest
+    needs: build
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        solc:
+          - "0.4.25"
+          - "0.5.7"
+        experimental: [false]
+        include:
+          - solc: "0.6.12"
+            experimental: true
+          - solc: "0.7.5"
+            experimental: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.6'
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/install-solc.sh
+          .github/scripts/install-crytic-compile.sh
+        env:
+          HOST_OS: ${{ runner.os }}
+          SOLC_VER: ${{ matrix.solc }}
+
+      - name: Download testsuite
+        uses: actions/download-artifact@v2
+        with:
+         name: echidna-testsuite
+
+      - name: Test
+        run: |
+          export PATH="${PATH}:$HOME/.local/bin"
+          cp "$HOME/.local/bin/solc-${{ matrix.solc }}" "$HOME/.local/bin/solc"
+          chmod +x echidna-testsuite
+          ./echidna-testsuite
diff --git a/package.yaml b/package.yaml
@@ -84,3 +84,11 @@ tests:
       - tasty
       - tasty-hunit
       - tasty-quickcheck
+    when:
+        - condition: os(linux)
+          ghc-options:
+            - -threaded
+            - -static
+            - -O2
+          cc-options: -static
+          ld-options: -static -pthread