qns.yml

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

name: qns

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  RUSTUP_MAX_RETRIES: 10
  RUST_BACKTRACE: 1
  # This kept breaking builds so we're pinning for now. We should do our best to keep
  # up with the changes, though.
  INTEROP_RUNNER_REF: 4be6491794a08899f295dc5cdf9eeba8e9fa5431
  # This should be updated when updating wesleyrosenblum/quic-network-simulator
  NETWORK_SIMULATOR_REF: sha256:20abe0bed8c0e39e1d8750507b24295f7c978bdd7e05fa6f3a5afed4b76dc191
  IPERF_ENDPOINT_REF: sha256:cb50cc8019d45d9cad5faecbe46a3c21dd5e871949819a5175423755a9045106
  WIRESHARK_VERSION: 3.7.1
  CDN: https://dnglbrstg7yg.cloudfront.net
  LOG_URL: logs/latest/SERVER_CLIENT/TEST/index.html

# By default depandabot only receives read permissions. Explicitly give it write
# permissions which is needed by the ouzi-dev/commit-status-updater task.
#
# Updating status is relatively safe (doesnt modify source code) and caution
# should we taken before adding more permissions.
permissions:
  statuses: write

jobs:
  env:
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.implementations.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
        with:
          path: s2n-quic

      - uses: actions/checkout@v4
        with:
          repository: marten-seemann/quic-interop-runner
          ref: ${{ env.INTEROP_RUNNER_REF }}
          path: quic-interop-runner

      - name: Patch quic-interop-runner
        working-directory: quic-interop-runner
        run: |
          git apply --3way ../s2n-quic/.github/interop/runner.patch

      - name: Define implementations
        id: implementations
        working-directory: quic-interop-runner
        run: |
          CLIENTS=$(cat implementations.json \
            | jq -c '[. | to_entries[] | select(.value.role == "both" or .value.role == "client") | {"client": .key, "server": "s2n-quic"}] | sort'
          )
          echo "Clients: $CLIENTS"
          SERVERS=$(cat implementations.json \
            | jq -c '[. | to_entries[] | select(.value.role == "both" or .value.role == "server") | {"client": "s2n-quic", "server": .key}] | sort'
          )
          echo "Servers: $SERVERS"
          MATRIX=$(echo "[$CLIENTS, $SERVERS]" | jq -c '{"include": . | flatten | sort | unique}')
          echo "Matrix: $MATRIX"
          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT

  s2n-quic-qns:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        mode: ["debug", "release"]
    # enable debug information
    env:
      RUSTFLAGS: "-g --cfg s2n_internal_dev --cfg s2n_quic_dump_on_panic --cfg s2n_quic_unstable"
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Install rust toolchain
        id: toolchain
        run: |
          rustup toolchain install stable --profile minimal
          rustup override set stable

      - uses: camshaft/rust-cache@v1
        with:
          key: ${{ matrix.mode }}-${{ env.RUSTFLAGS }}

      - name: Run cargo build
        run: cargo build --bin s2n-quic-qns ${{ matrix.mode == 'release' && '--release' || '' }}

      - name: Prepare artifact
        run: |
          mkdir -p s2n-quic-qns
          cp target/${{ matrix.mode }}/s2n-quic-qns s2n-quic-qns/s2n-quic-qns-${{ matrix.mode }}

      - uses: actions/upload-artifact@v4
        with:
          name: s2n-quic-qns-${{ matrix.mode }}
          path: s2n-quic-qns/

  interop:
    runs-on: ubuntu-22.04
    needs: [env, s2n-quic-qns]
    strategy:
      matrix: ${{ fromJson(needs.env.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4
        with:
          path: s2n-quic

      - uses: actions/download-artifact@v4
        with:
          name: s2n-quic-qns-debug
          path: s2n-quic-qns/

      - uses: actions/download-artifact@v4
        with:
          name: s2n-quic-qns-release
          path: s2n-quic-qns/

      - name: Setup dockerfile
        working-directory: s2n-quic-qns
        run: |
          cp ../s2n-quic/quic/s2n-quic-qns/etc/Dockerfile .
          cp ../s2n-quic/quic/s2n-quic-qns/etc/run_endpoint.sh .

      - name: Run docker build
        working-directory: s2n-quic-qns
        env:
          DOCKER_BUILDKIT: 1
        run: |
          docker build . --file Dockerfile --target prebuilt --tag aws/s2n-quic-qns --build-arg tls=s2n-tls
          docker build . --file Dockerfile --target prebuilt --tag aws/s2n-quic-qns-rustls --build-arg tls=rustls

      - uses: actions/checkout@v4
        with:
          repository: marten-seemann/quic-interop-runner
          ref: ${{ env.INTEROP_RUNNER_REF }}
          path: quic-interop-runner

      - name: Patch quic-interop-runner
        working-directory: quic-interop-runner
        run: |
          git apply --3way ../s2n-quic/.github/interop/runner.patch

      - name: Run docker pull
        working-directory: quic-interop-runner
        run: |
          docker pull "wesleyrosenblum/quic-network-simulator@$NETWORK_SIMULATOR_REF"
          docker pull "martenseemann/quic-interop-iperf-endpoint@$IPERF_ENDPOINT_REF"

      - uses: actions/setup-python@v5
        with:
          python-version: 3.7

      - name: Install tshark
        run: |
          wget --no-verbose https://dnglbrstg7yg.cloudfront.net/tshark/v$WIRESHARK_VERSION/tshark
          chmod +x tshark
          sudo mv tshark /usr/bin
          /usr/bin/tshark -v

      - name: Install dependencies
        working-directory: quic-interop-runner
        run: |
          python3 -m pip install --upgrade pip
          pip3 install wheel
          pip3 install --upgrade -r requirements.txt

      - name: Run quic-interop-runner
        working-directory: quic-interop-runner
        run: |
          # enable IPv6 support
          sudo modprobe ip6table_filter
          python3 run.py --client ${{ matrix.client }} --server ${{ matrix.server }} --json results/result.json --debug --log-dir results/logs
          mkdir -p results/logs

      - name: Prepare artifacts
        working-directory: quic-interop-runner
        run: |
          ls -al results
          # clean up invalid path characters
          find results -name '*:*' | while read from; do
            echo "Invalid filename: $from"
            to=$(echo $from | sed 's/:/_/g')
            mv $from $to
          done
          # remove files we don't do anything with to reduce the artifact size
          find results -name '*.qlog' -exec rm {} \;
          # remove cross traffic and goodput packet captures as they are large and not useful
          find results -maxdepth 7 -type d -path "**/crosstraffic/*/sim" | xargs rm -rf \;
          find results -maxdepth 7 -type d -path "**/goodput/*/sim" | xargs rm -rf \;

          # Add index files for easy browsing
          find results -maxdepth 3 -type d -path "*/logs/*/*" | while read from; do
              tree -H "." \
                -h \
                -L 3 \
                -I 'index.html' \
                -T "${{ matrix.client }} client / ${{ matrix.server }} server - $(basename $from)" \
                --noreport \
                --charset utf-8 \
                -o $from/index.html \
                $from
          done

      - uses: aws-actions/configure-aws-credentials@v4.0.2
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-1

      - name: Upload to S3
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        id: s3
        working-directory: quic-interop-runner
        run: |
          TARGET="${{ github.sha }}/interop/logs/latest"
          aws s3 sync results/logs "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks

      - uses: actions/upload-artifact@v4
        with:
          name: interop-${{ matrix.client }}-client-${{ matrix.server }}-server
          path: quic-interop-runner/results/result.json

      - name: Assert no crashes
        working-directory: quic-interop-runner
        run: |
          ! grep -Rq 'The s2n-quic-qns application shut down unexpectedly' results

  interop-report:
    runs-on: ubuntu-22.04
    needs: [interop]
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          path: results/

      - name: Download latest results
        id: download
        run: |
          rm -f result.json
          INTEROP_BASE_URL="https://interop.seemann.io/logs/"
          wget ${INTEROP_BASE_URL}latest/result.json || echo '{}' > result.json
          mv result.json latest.json
          INTEROP_LOG_URL=${INTEROP_BASE_URL}$(jq --raw-output '.log_dir' latest.json)/SERVER_CLIENT/TEST/
          echo "INTEROP_LOG_URL=$INTEROP_LOG_URL" >> $GITHUB_OUTPUT

      - name: Get latest successful interop commit SHA on main branch
        id: mainsha
        if: github.event.pull_request
        run: |
          curl \
          --url "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/qns.yml/runs?branch=main&status=success&per_page=1" \
          --header "Accept: application/vnd.github.v3+json" > latest_workflow_run.json
          MAIN_SHA=$(jq --raw-output '.workflow_runs[0] | .head_sha' latest_workflow_run.json)
          rm -f latest_workflow_run.json
          echo "MAIN_SHA=$MAIN_SHA" >> $GITHUB_OUTPUT

      - name: Download latest main interop result
        if: github.event.pull_request
        run: |
          rm -f prev_result.json
          wget $CDN/${{ steps.mainsha.outputs.MAIN_SHA }}/interop/logs/latest/result.json || echo '{}' > result.json
          mv result.json prev_result.json

      - name: Generate report for pull request
        if: github.event.pull_request
        run: |
          mkdir -p web/logs/latest
          MAIN_SHA=${{ steps.mainsha.outputs.MAIN_SHA }}
          python3 .github/interop/merge.py \
            --prev_version prev_result.json \
            --new_version_suffix "pr${{github.event.pull_request.number}}" \
            --new_version_url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/${{github.event.pull_request.number}}" \
            --new_version_log_url "$LOG_URL" \
            --prev_version_log_url "$CDN/$MAIN_SHA/interop/$LOG_URL" \
            --prev_version_url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/tree/$MAIN_SHA" \
            --interop_log_url "${{ steps.download.outputs.INTEROP_LOG_URL }}" \
            latest.json \
            results/**/result.json > \
              web/logs/latest/result.json

      - name: Generate report for push to main
        if: github.event_name == 'push'
        run: |
          mkdir -p web/logs/latest
          python3 .github/interop/merge.py \
            --new_version_log_url "$LOG_URL" \
            --new_version_url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/tree/$GITHUB_SHA" \
            --interop_log_url "${{ steps.download.outputs.INTEROP_LOG_URL }}" \
            latest.json \
            results/**/result.json > \
              web/logs/latest/result.json

      - uses: aws-actions/configure-aws-credentials@v4.0.2
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-1

      - name: Upload to S3
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        id: s3
        run: |
          cp .github/interop/*.html web/
          cp .github/interop/*.js web/
          TARGET="${{ github.sha }}/interop"
          aws s3 sync web "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks
          URL="$CDN/$TARGET/index.html"
          echo "URL=$URL" >> $GITHUB_OUTPUT

      - uses: ouzi-dev/commit-status-updater@v2.0.2
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        with:
          name: "interop / report"
          status: "success"
          url: "${{ steps.s3.outputs.URL }}"

      - name: Check for regressions
        run: |
          python3 .github/interop/check.py \
            --required .github/interop/required.json \
            web/logs/latest/result.json

  bench:
    runs-on: ubuntu-22.04
    needs: [env, s2n-quic-qns]
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Install rust toolchain
        id: toolchain
        run: |
          rustup toolchain install stable
          rustup override set stable

      - uses: camshaft/rust-cache@v1

      - name: Install tshark
        run: |
          wget --no-verbose https://dnglbrstg7yg.cloudfront.net/tshark/v$WIRESHARK_VERSION/tshark
          chmod +x tshark
          sudo mv tshark /usr/bin
          /usr/bin/tshark -v

      - name: Install gnuplot
        run: |
          sudo apt-get -o Acquire::Retries=3 update
          sudo apt-get -o Acquire::Retries=3 install -y gnuplot

      # authenticate pull to avoid hitting pull quota
      - name: Login to Amazon Elastic Container Registry Public
        if: github.repository == github.event.pull_request.head.repo.full_name
        uses: docker/login-action@v3.3.0
        with:
          registry: public.ecr.aws
          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

      - name: Pull s2n-quic-qns:main
        if: github.event.pull_request
        run: docker pull public.ecr.aws/s2n/s2n-quic-qns:main

      - uses: actions/download-artifact@v4
        with:
          name: s2n-quic-qns-debug
          path: s2n-quic-qns-build/

      - uses: actions/download-artifact@v4
        with:
          name: s2n-quic-qns-release
          path: s2n-quic-qns-build/

      - name: Setup dockerfile
        working-directory: s2n-quic-qns-build
        run: |
          cp ../quic/s2n-quic-qns/etc/Dockerfile .
          cp ../quic/s2n-quic-qns/etc/run_endpoint.sh .

      - name: Run docker build
        working-directory: s2n-quic-qns-build
        env:
          DOCKER_BUILDKIT: 1
        run: |
          docker build . --file Dockerfile --target prebuilt --tag aws/s2n-quic-qns

      - name: Run script for pull request
        if: github.event.pull_request
        run: sudo env "PATH=$PATH" "BUILD_S2N_QUIC=false" "COMPARE_TO_MAIN=true" ./scripts/benchmark/run-all

      - name: Run script for push to main
        if: github.event_name == 'push'
        run: sudo env "PATH=$PATH" "BUILD_S2N_QUIC=false" ./scripts/benchmark/run-all

      - uses: aws-actions/configure-aws-credentials@v4.0.2
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-1

      - name: Upload results
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        id: s3
        run: |
          TARGET="${{ github.sha }}/bench"
          aws s3 sync target/benchmark/results "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks
          URL="$CDN/$TARGET/index.html"
          echo "URL=$URL" >> $GITHUB_OUTPUT

      - uses: ouzi-dev/commit-status-updater@v2.0.2
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        with:
          name: "bench / report"
          status: "success"
          url: "${{ steps.s3.outputs.URL }}"

      - name: Assert no crashes
        run: |
          ! grep -Rq 'The s2n-quic-qns application shut down unexpectedly' target/benchmark/results

  h3spec:
    runs-on: ubuntu-22.04
    needs: [s2n-quic-qns]
    strategy:
      matrix:
        tls: ["s2n-tls", "rustls"]

    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: s2n-quic-qns-debug

      - name: Run test
        run: |
          chmod +x s2n-quic-qns-debug
          ./s2n-quic-qns-debug interop server --port 4433 --tls ${{ matrix.tls }} &
          # wait for the server to boot
          sleep 3
          ./scripts/h3spec

  perf:
    runs-on: ubuntu-22.04
    needs: [s2n-quic-qns]
    strategy:
      matrix:
        include:
          - client: "s2n-quic"
            server: "s2n-quic"
          - client: "s2n-quic-null"
            server: "s2n-quic-null"

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Install rust toolchain
        id: toolchain
        run: |
          rustup toolchain install stable --profile minimal
          rustup override set stable

      - name: Install perf
        run: |
          sudo apt-get -y update
          sudo apt-get install -y linux-tools-$(uname -r) linux-tools-generic

      - name: Install inferno
        uses: camshaft/install@v1
        with:
          crate: inferno
          bins: inferno-collapse-perf,inferno-flamegraph

      - name: Install ultraman
        uses: camshaft/install@v1
        with:
          crate: ultraman

      - uses: actions/download-artifact@v4
        with:
          name: s2n-quic-qns-release
          path: target/release/

      - name: Setup artifacts
        run: |
          mv target/release/s2n-quic-qns-release target/release/s2n-quic-qns
          chmod +x target/release/s2n-quic-qns

      - name: Run script
        env:
          # ultraman wants a SHELL var to spawn tasks
          SHELL: /bin/bash
        run: |
          set -e

          # set larger socket buffers
          sudo sysctl -w net.core.wmem_default=2000000
          sudo sysctl -w net.core.rmem_default=2000000

          mkdir -p target/perf/results
          sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 10000 0 ${{ matrix.server }} ${{ matrix.client }}
          sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 7500 2500 ${{ matrix.server }} ${{ matrix.client }}
          sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 5000 5000 ${{ matrix.server }} ${{ matrix.client }}
          sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 2500 7500 ${{ matrix.server }} ${{ matrix.client }}
          sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 0 10000 ${{ matrix.server }} ${{ matrix.client }}
          sudo chown -R $(whoami) target/perf/results

      - name: Prepare artifacts
        run: |
          cd ./target/perf/results
          zip perf.zip **/*.script
          rm -rf **/*.script

      - uses: actions/upload-artifact@v4
        with:
          name: perf-results-${{ matrix.server }}-${{ matrix.client }}
          path: target/perf/results

  perf-report:
    runs-on: ubuntu-22.04
    needs: [perf]
    steps:
      - uses: actions/checkout@v4

      # add any additional perf tests here
      - uses: actions/download-artifact@v4
        with:
          name: perf-results-s2n-quic-s2n-quic
          path: perf-results/
      - uses: actions/download-artifact@v4
        with:
          name: perf-results-s2n-quic-null-s2n-quic-null
          path: perf-results/

      - name: Generate report
        run: |
          cd perf-results
          tree -H "." -T "Performance Results" --noreport --charset utf-8 > index.html

      - uses: aws-actions/configure-aws-credentials@v4.0.2
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-1

      - name: Upload results
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        id: s3
        run: |
          TARGET="${{ github.sha }}/perf"
          aws s3 sync perf-results "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks
          URL="$CDN/$TARGET/index.html"
          echo "URL=$URL" >> $GITHUB_OUTPUT

      - uses: ouzi-dev/commit-status-updater@v2.0.2
        if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name
        with:
          name: "perf / report"
          status: "success"
          url: "${{ steps.s3.outputs.URL }}"

  attack:
    runs-on: ubuntu-22.04
    needs: [s2n-quic-qns]
    strategy:
      matrix:
        attack: ["udp"]

    steps:
      - uses: actions/checkout@v4

      - name: Install rust toolchain
        id: toolchain
        run: |
          rustup toolchain install stable --profile minimal
          rustup override set stable

      - uses: actions/download-artifact@v4
        with:
          name: s2n-quic-qns-debug

      - name: Run cargo build
        working-directory: tools/${{ matrix.attack }}-attack
        run: cargo +stable build --release

      - name: Start client
        working-directory: tools/${{ matrix.attack }}-attack
        run: |
          ./target/release/${{ matrix.attack }}-attack localhost:4433 &

      - name: Start server
        shell: bash
        run: |
          chmod +x ./s2n-quic-qns-debug
          # disable exiting on errors to capture the timeout status
          set +e

          timeout 5m ./s2n-quic-qns-debug interop server --port 4433
          EXIT_CODE="$?"

          set -e
          # `timeout` exits with `124` if the time limit was reached
          [[ "$EXIT_CODE" == "124" ]] || exit $EXIT_CODE