benchmark.yml

name: Benchmark

on:
  schedule:
    # Run at 03:00 clock UTC on Monday and Wednesday
    - cron: "0 03 * * 1,3"
  pull_request:
    paths:
      - '.github/workflows/benchmark.yml'
  workflow_dispatch:

env:
  CARGO_TERM_COLOR: always

jobs:
  contrib-build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup Golang
      uses: actions/setup-go@v5
      with:
        go-version-file: 'go.work'
        cache-dependency-path: "**/*.sum"
    - name: Build Contrib
      run: |
        make -e DOCKER=false nydusify-release
    - name: Upload Nydusify
      uses: actions/upload-artifact@v4
      with:
        name: nydusify-artifact
        path: contrib/nydusify/cmd/nydusify

  nydus-build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Rust Cache
      uses: Swatinem/rust-cache@v2
      with:
        cache-on-failure: true
        shared-key: Linux-cargo-amd64
    - uses: dsherret/rust-toolchain-file@v1
    - name: Build Nydus
      run: |
        make release
    - name: Upload Nydus Binaries
      uses: actions/upload-artifact@v4
      with:
        name: nydus-artifact
        path: |
          target/release/nydus-image
          target/release/nydusd

  benchmark-description:
    runs-on: ubuntu-latest
    steps:
    - name: Description
      run: |
        echo "## Benchmark Environment" > $GITHUB_STEP_SUMMARY
        echo "| operating system | cpu | memory " >> $GITHUB_STEP_SUMMARY
        echo "|:----------------:|:---:|:------ " >> $GITHUB_STEP_SUMMARY
        echo "| ubuntu-22.04 | 2-core CPU (x86_64) | 7GB |" >> $GITHUB_STEP_SUMMARY

  benchmark-oci:
    runs-on: ubuntu-latest
    needs: [contrib-build, nydus-build]
    strategy:
      matrix:
        include:
          - image: wordpress
            tag: 6.1.1
          - image: node
            tag: 19.8
          - image: python
            tag: 3.10.7
          - image: golang
            tag: 1.19.3
          - image: ruby
            tag: 3.1.3
          - image: amazoncorretto
            tag: 8-al2022-jdk
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Download Nydus
      uses: actions/download-artifact@v4
      with:
        name: nydus-artifact
        path: target/release
    - name: Download Nydusify
      uses: actions/download-artifact@v4
      with:
        name: nydusify-artifact
        path: contrib/nydusify/cmd
    - name: Prepare Environment
      run: |
        sudo bash misc/prepare.sh
    - name: BenchMark Test
      run: |
        export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }}
        export BENCHMARK_MODE=oci
        export BENCHMARK_METRIC_FILE=${{ matrix.image }}-oci.json
        export SNAPSHOTTER=overlayfs
        sudo -E make smoke-benchmark
    - name: Save BenchMark Result
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-oci-${{ matrix.image }}
        path: smoke/${{ matrix.image }}-oci.json

  benchmark-fsversion-v5:
    runs-on: ubuntu-latest
    needs: [contrib-build, nydus-build]
    strategy:
      matrix:
        include:
          - image: wordpress
            tag: 6.1.1
          - image: node
            tag: 19.8
          - image: python
            tag: 3.10.7
          - image: golang
            tag: 1.19.3
          - image: ruby
            tag: 3.1.3
          - image: amazoncorretto
            tag: 8-al2022-jdk
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Download Nydus
      uses: actions/download-artifact@v4
      with:
        name: nydus-artifact
        path: target/release
    - name: Download Nydusify
      uses: actions/download-artifact@v4
      with:
        name: nydusify-artifact
        path: contrib/nydusify/cmd
    - name: Prepare Environment
      run: |
        sudo bash misc/prepare.sh
    - name: BenchMark Test
      run: |
        export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }}
        export BENCHMARK_MODE=fs-version-5
        export BENCHMARK_METRIC_FILE=${{ matrix.image }}-fsversion-v5.json
        sudo -E make smoke-benchmark
    - name: Save BenchMark Result
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-fsversion-v5-${{ matrix.image }}
        path: smoke/${{ matrix.image }}-fsversion-v5.json

  benchmark-fsversion-v6:
    runs-on: ubuntu-latest
    needs: [contrib-build, nydus-build]
    strategy:
      matrix:
        include:
          - image: wordpress
            tag: 6.1.1
          - image: node
            tag: 19.8
          - image: python
            tag: 3.10.7
          - image: golang
            tag: 1.19.3
          - image: ruby
            tag: 3.1.3
          - image: amazoncorretto
            tag: 8-al2022-jdk
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Download Nydus
      uses: actions/download-artifact@v4
      with:
        name: nydus-artifact
        path: target/release
    - name: Download Nydusify
      uses: actions/download-artifact@v4
      with:
        name: nydusify-artifact
        path: contrib/nydusify/cmd
    - name: Prepare Environment
      run: |
        sudo bash misc/prepare.sh
    - name: BenchMark Test
      run: |
        export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }}
        export BENCHMARK_MODE=fs-version-6
        export BENCHMARK_METRIC_FILE=${{ matrix.image }}-fsversion-v6.json
        sudo -E make smoke-benchmark
    - name: Save BenchMark Result
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-fsversion-v6-${{ matrix.image }}
        path: smoke/${{ matrix.image }}-fsversion-v6.json

  benchmark-zran:
    runs-on: ubuntu-latest
    needs: [contrib-build, nydus-build]
    strategy:
      matrix:
        include:
          - image: wordpress
            tag: 6.1.1
          - image: node
            tag: 19.8
          - image: python
            tag: 3.10.7
          - image: golang
            tag: 1.19.3
          - image: ruby
            tag: 3.1.3
          - image: amazoncorretto
            tag: 8-al2022-jdk
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Download Nydus
      uses: actions/download-artifact@v4
      with:
        name: nydus-artifact
        path: target/release
    - name: Download Nydusify
      uses: actions/download-artifact@v4
      with:
        name: nydusify-artifact
        path: contrib/nydusify/cmd
    - name: Prepare Environment
      run: |
        sudo bash misc/prepare.sh
    - name: BenchMark Test
      run: |
        export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }}
        export BENCHMARK_MODE=zran
        export BENCHMARK_METRIC_FILE=${{ matrix.image }}-zran.json
        sudo -E make smoke-benchmark
    - name: Save BenchMark Result
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-zran-${{ matrix.image }}
        path: smoke/${{ matrix.image }}-zran.json

  benchmark-result:
    runs-on: ubuntu-latest
    needs: [benchmark-oci, benchmark-fsversion-v5, benchmark-fsversion-v6, benchmark-zran]
    strategy:
      matrix:
        include:
          - image: wordpress
            tag: 6.1.1
          - image: node
            tag: 19.8
          - image: python
            tag: 3.10.7
          - image: golang
            tag: 1.19.3
          - image: ruby
            tag: 3.1.3
          - image: amazoncorretto
            tag: 8-al2022-jdk
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Download benchmark-oci
      uses: actions/download-artifact@v4
      with:
        name: benchmark-oci-${{ matrix.image }}
        path: benchmark-result
    - name: Download benchmark-fsversion-v5
      uses: actions/download-artifact@v4
      with:
        name: benchmark-fsversion-v5-${{ matrix.image }}
        path: benchmark-result
    - name: Download benchmark-fsversion-v6
      uses: actions/download-artifact@v4
      with:
        name: benchmark-fsversion-v6-${{ matrix.image }}
        path: benchmark-result
    - name: Download benchmark-zran
      uses: actions/download-artifact@v4
      with:
        name: benchmark-zran-${{ matrix.image }}
        path: benchmark-result
    - name: Benchmark Summary
      run: |
        case ${{matrix.image}} in
          "wordpress")
              echo "### workload: wait the 80 port response" > $GITHUB_STEP_SUMMARY
            ;;
          "node")
              echo "### workload: node index.js; wait the 80 port response" > $GITHUB_STEP_SUMMARY
            ;;
          "python")
              echo "### workload: python -c 'print("hello")'" > $GITHUB_STEP_SUMMARY
            ;;
          "golang")
              echo "### workload: go run main.go" > $GITHUB_STEP_SUMMARY
            ;;
          "ruby")
              echo "### workload: ruby -e "puts \"hello\""" > $GITHUB_STEP_SUMMARY
            ;;
          "amazoncorretto")
              echo "### workload: javac Main.java; java Main" > $GITHUB_STEP_SUMMARY
            ;;
        esac
        cd benchmark-result
        metric_files=(
          "${{ matrix.image }}-oci.json"
          "${{ matrix.image }}-fsversion-v5.json"
          "${{ matrix.image }}-fsversion-v6.json"
          "${{ matrix.image }}-zran.json"
        )
        echo "| bench-result | e2e-time(s) | read-count | read-amount(MB) | image-size(MB) |convert-time(s)|" >> $GITHUB_STEP_SUMMARY
        echo "|:-------------|:-----------:|:----------:|:---------------:|:--------------:|:-------------:|" >> $GITHUB_STEP_SUMMARY
        for file in "${metric_files[@]}"; do
            name=$(basename "$file" .json | sed 's/^[^-]*-\(.*\)$/\1/')
            data=$(jq -r '. | "\(.e2e_time / 1e9) \(.read_count) \(.read_amount_total / (1024 * 1024)) \(.image_size / (1024 * 1024)) \(.conversion_elapsed / 1e9)"' "$file" | \
              awk '{ printf "%.2f | %.0f | %.2f | %.2f | %.2f", $1, $2, $3, $4, $5 }')
            echo "| $name | $data |" >> $GITHUB_STEP_SUMMARY
        done