diff --git a/.github/workflows/report_test_flakes.yaml b/.github/workflows/report_test_flakes.yaml new file mode 100644 index 000000000..78dd37ee8 --- /dev/null +++ b/.github/workflows/report_test_flakes.yaml @@ -0,0 +1,125 @@ +on: + workflow_call: + +name: 'Workflow Analysis' + +jobs: + report-test-flakes: + name: 'Report test flakes' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/download-artifact@v3 + - name: Parse test results + id: parse-test-results + run: | + sudo apt-get install colorized-logs + + echo -n 'results=[' >> $GITHUB_OUTPUT + + ENTRIES="" + + for RESULTS_DIR in test-results-*/ ; do + mapfile -t target <$RESULTS_DIR/target + + PLATFORM="${target[0]}" + FEATURES="${target[1]}" + TOOLCHAIN="${target[2]}" + + LOG_PATH="$RESULTS_DIR/log" + csplit -q "$LOG_PATH" %^------------% + + SUMMARY="" + + if [[ -f "./xx00" ]]; then + SUMMARY=$(tail ./xx00 -n+2 | ansi2txt | jq -M --compact-output --raw-input --slurp . | sed -e 's/\\/\\\\/g') + else + continue + fi + + ENTRY="{\"platform\":\"$PLATFORM\",\"features\":\"$FEATURES\",\"toolchain\":\"$TOOLCHAIN\",\"summary\":$SUMMARY}" + + if [ -z "$ENTRIES" ]; then + ENTRIES="$ENTRY" + else + ENTRIES="$ENTRIES,$ENTRY" + fi + + done + + echo -n "$ENTRIES ]" >> $GITHUB_OUTPUT + - name: Report test flakes + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const results = JSON.parse(`${{ steps.parse-test-results.outputs.results }}`); + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const testFlakeAnalysisHeader = 'Test flake analysis'; + const existingComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes(testFlakeAnalysisHeader) + }); + + let body = ''; + + if (results.length == 0) { + body = "No test results to analyze. Maybe none of the test runs passed?"; + } else { + + let table = "\n\n| status | platform | features | toolchain |\n|---|---|---|---|\n"; + + const flakeSummaries = []; + + for (result of results) { + const isFlakey = result.summary.indexOf("FLAKY") > 0; + + table += `| ${ isFlakey ? "🟡" : "🟢" } | \`${ result.platform }\` | \`${ result.features }\` | \`${result.toolchain}\` |\n`; + + if (isFlakey) { + flakeSummaries.push(`#### Flake summary for \`${ result.platform }\`, \`${ result.features }\`, \`${ result.toolchain }\` + + \`\`\`shell + ${ result.summary } + \`\`\``); + } + } + + if (flakeSummaries.length == 0) { + body += '\nNo flakes detected 🎉\n\n' + } + + body += table; + + if (flakeSummaries.length > 0) { + body += "\n\n"; + body += flakeSummaries.join('\n\n'); + } + } + + body = `### ${testFlakeAnalysisHeader} + + ${body}`; + + if (existingComment) { + github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body + }); + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body + }); + } diff --git a/.github/workflows/run_test_suite.yaml b/.github/workflows/run_test_suite.yaml index 1ffae7db5..452b5a7b5 100644 --- a/.github/workflows/run_test_suite.yaml +++ b/.github/workflows/run_test_suite.yaml @@ -56,86 +56,73 @@ jobs: - name: 'Run Linter' run: cargo clippy --all -- -D warnings - run-test-suite-windows: - runs-on: windows-latest + run-rust-test-suite: + name: 'Run Rust test suite' + strategy: + matrix: + features: ['test-kubo,headers', 'test-kubo,headers,rocksdb'] + platform: ['ubuntu-latest', 'windows-latest', 'macos-13'] + toolchain: ['stable'] + exclude: + - platform: 'windows-latest' + features: 'test-kubo,headers,rocksdb' + runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - name: 'Setup Rust' run: | curl -sSf https://sh.rustup.rs | sh -s -- -y - - name: 'Install environment packages' + rustup toolchain install ${{matrix.toolchain}} + - name: 'Install environment packages (Windows)' + if: ${{ matrix.platform == 'windows-latest' }} run: | choco install -y cmake protoc openssl shell: sh - - name: Install cargo-binstall - uses: cargo-bins/cargo-binstall@v1.4.4 - - name: Install binaries from cargo - run: | - cargo binstall cargo-nextest --no-confirm - - name: 'Install IPFS Kubo' - uses: ibnesayeed/setup-ipfs@master - with: - ipfs_version: v0.17.0 - run_daemon: true - - name: 'Run Rust native target tests' - run: cargo nextest run --features test-kubo,headers --retries 5 --color always - env: - NOOSPHERE_LOG: academic - - run-test-suite-linux: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: 'Setup Rust' - run: | - curl -sSf https://sh.rustup.rs | sh -s -- -y - - name: 'Install environment packages' + - name: 'Install environment packages (Linux)' + if: ${{ matrix.platform == 'ubuntu-latest' }} run: | sudo apt-get update -qqy sudo apt-get install jq protobuf-compiler cmake - - name: Install cargo-binstall - uses: cargo-bins/cargo-binstall@v1.4.4 - - name: Install binaries from cargo - run: | - cargo binstall cargo-nextest --no-confirm - name: 'Install IPFS Kubo' uses: ibnesayeed/setup-ipfs@master with: ipfs_version: v0.17.0 run_daemon: true - - name: 'Run Rust native target tests' - run: cargo nextest run --features test-kubo,headers --retries 5 --color always - env: - NOOSPHERE_LOG: academic - - run-test-suite-linux-rocksdb: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: 'Setup Rust' - run: | - curl -sSf https://sh.rustup.rs | sh -s -- -y - - name: 'Install environment packages' - run: | - sudo apt-get update -qqy - sudo apt-get install jq protobuf-compiler cmake libclang-dev - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.4.4 - name: Install binaries from cargo run: | - cargo binstall cargo-nextest --no-confirm - - name: 'Install IPFS Kubo' - uses: ibnesayeed/setup-ipfs@master - with: - ipfs_version: v0.17.0 - run_daemon: true - - name: 'Run Rust native target tests (RocksDB)' - run: cargo nextest run --features rocksdb,test-kubo,headers --retries 5 --color always + cargo +${{ matrix.toolchain }} binstall cargo-nextest --no-confirm --force + - name: 'Run Rust tests' + shell: bash + run: | + mkdir -p test-results + + echo "${{ matrix.platform }} + ${{ matrix.features }} + ${{ matrix.toolchain}}" > test-results/target + + FEATURES=""; + + if [ -n "${{matrix.features}}" ]; then + FEATURES="--features ${{matrix.features}}" + fi + + cargo +${{ matrix.toolchain }} nextest run $FEATURES --retries 5 --color always 2>&1 | tee test-results/log env: NOOSPHERE_LOG: academic + - uses: actions/upload-artifact@v3 + with: + name: test-results-${{ hashFiles('./test-results') }} + path: ./test-results + + report-test-flakes: + name: 'Report test flakes (Linux)' + needs: ['run-rust-test-suite'] + if: always() + uses: ./.github/workflows/report_test_flakes.yaml + secrets: inherit run-test-suite-linux-c: runs-on: ubuntu-latest