feat: multithreading proof generator #31
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - 'master' | |
| tags: | |
| - '**' | |
| jobs: | |
| build-linux: | |
| runs-on: [ "self-hosted", "aws_autoscaling"] | |
| env: | |
| CONTAINER_TMP: /opt/ | |
| HOST_TMP: /home/runner/work/_temp/ | |
| DEBIAN_FRONTEND: noninteractive | |
| BOOST_VERSION: "1.80.0" | |
| container: | |
| image: ubuntu:22.04 | |
| volumes: | |
| - /home/runner/work/_temp/:/opt/ | |
| steps: | |
| - name: Install dependencies | |
| run: | | |
| env && \ | |
| apt update && \ | |
| apt install -y \ | |
| build-essential \ | |
| libssl-dev \ | |
| cmake \ | |
| git \ | |
| unzip \ | |
| libicu-dev \ | |
| curl \ | |
| pkg-config \ | |
| wget \ | |
| \ | |
| libspdlog-dev \ | |
| liblz4-dev \ | |
| libgnutls28-dev \ | |
| libprotobuf-dev \ | |
| libyaml-cpp-dev \ | |
| libsctp-dev \ | |
| ragel \ | |
| xfslibs-dev \ | |
| systemtap-sdt-dev \ | |
| libc-ares-dev \ | |
| libhwloc-dev | |
| # Everything will be cleaned on checkout anyway | |
| wget http://launchpadlibrarian.net/466750967/libfmt-dev_6.1.2+ds-2_amd64.deb | |
| apt remove -y libfmt-dev | |
| apt install -y ./libfmt-dev_6.1.2+ds-2_amd64.deb | |
| - name: Print toolchain information | |
| run: | | |
| git --version | |
| cc --version | |
| cmake --version | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| # Workaround: https://github.com/actions/checkout/issues/1169 | |
| - name: Mark directory as safe | |
| run: | | |
| git config --system --add safe.directory $PWD | |
| - name: Install boost | |
| uses: MarkusJx/install-boost@v2.4.1 | |
| id: install-boost | |
| with: | |
| # A list of supported versions can be found here: | |
| # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json | |
| boost_version: ${{ env.BOOST_VERSION }} | |
| boost_install_dir: ${{ env.CONTAINER_TMP }} | |
| platform_version: 22.04 | |
| toolset: gcc | |
| arch: x86 | |
| - name: Set VERSION variable | |
| # required for building deb packages | |
| # TODO: replace with getting version from git tag or commit SHA1 | |
| run: echo "VERSION=$(cat VERSION)" >> $GITHUB_ENV | |
| - name: Configure CMake | |
| env: | |
| BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}" | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -B build \ | |
| -DCMAKE_BUILD_TYPE=Release | |
| -DBUILD_WITH_NUMA=FALSE | |
| - name: Build proof-producer | |
| env: | |
| BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}" | |
| run: | | |
| cd build | |
| make | |
| cd .. | |
| - name: Create .deb package | |
| # always build deb packages to check that they're buildable | |
| # although, push them only on tags (see later) | |
| run: | | |
| mkdir -p .debpkg/usr/bin | |
| cp ./build/bin/proof-generator/proof-generator .debpkg/usr/bin | |
| chmod +x .debpkg/usr/bin/proof-generator | |
| mkdir -p .debpkg/DEBIAN | |
| - uses: jiro4989/build-deb-action@v3 | |
| with: | |
| package: proof-generator | |
| package_root: .debpkg | |
| maintainer: =nil; Foundation | |
| version: ${{ env.VERSION }} | |
| arch: 'amd64' | |
| desc: 'zkllvm proof-generator' | |
| - name: Install aws tools | |
| # these steps run only on tag push => deb package release | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip awscliv2.zip | |
| ./aws/install | |
| pip3 install mkrepo | |
| - name: Upload packages to repository | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') | |
| env: | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} | |
| run: | | |
| PROOF_GENERATOR_PACKAGE=`ls proof-generator*.deb | cut -d'/' -f2` | |
| POOL="ubuntu/pool/main/p/proof-generator" | |
| aws s3api put-object --bucket deb.nil.foundation --key $POOL/$PROOF_GENERATOR_PACKAGE --body $PROOF_GENERATOR_PACKAGE | |
| mkrepo s3://deb.nil.foundation/ubuntu/ |