chore: Report flakes in CI

.github/workflows/report_test_flakes.yaml

@@ -0,0 +1,125 @@
+on:
+  workflow_call:
+
+name: 'Workflow Analysis'
+
+jobs:
+  report-test-flakes:
+    name: 'Report test flakes'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Parse test results
+        id: parse-test-results
+        run: |
+          sudo apt-get install colorized-logs
+
+          echo -n 'results=[' >> $GITHUB_OUTPUT
+
+          ENTRIES=""
+
+          for RESULTS_DIR in test-results-*/ ; do
+            mapfile -t target <$RESULTS_DIR/target
+
+            PLATFORM="${target[0]}"
+            FEATURES="${target[1]}"
+            TOOLCHAIN="${target[2]}"
+
+            LOG_PATH="$RESULTS_DIR/log"
+            csplit -q "$LOG_PATH" %^------------%
+
+            SUMMARY=""
+
+            if [[ -f "./xx00" ]]; then
+              SUMMARY=$(tail ./xx00 -n+2 | ansi2txt | jq -M --compact-output --raw-input --slurp . | sed -e 's/\\/\\\\/g')
+            else
+              continue
+            fi
+
+            ENTRY="{\"platform\":\"$PLATFORM\",\"features\":\"$FEATURES\",\"toolchain\":\"$TOOLCHAIN\",\"summary\":$SUMMARY}"
+
+            if [ -z "$ENTRIES" ]; then
+              ENTRIES="$ENTRY"
+            else
+              ENTRIES="$ENTRIES,$ENTRY"
+            fi
+
+          done
+
+          echo -n "$ENTRIES ]" >> $GITHUB_OUTPUT
+      - name: Report test flakes
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const results = JSON.parse(`${{ steps.parse-test-results.outputs.results }}`);
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const testFlakeAnalysisHeader = 'Test flake analysis';
+            const existingComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes(testFlakeAnalysisHeader)
+            });
+
+            let body = '';
+
+            if (results.length == 0) {
+              body = "No test results to analyze. Maybe none of the test runs passed?";
+            } else {
+
+              let table = "\n\n| status | platform | features | toolchain |\n|---|---|---|---|\n";
+
+              const flakeSummaries = [];
+
+              for (result of results) {
+                const isFlakey = result.summary.indexOf("FLAKY") > 0;
+
+                table += `| ${ isFlakey ? "🟡" : "🟢" } | \`${ result.platform }\` | \`${ result.features }\` | \`${result.toolchain}\` |\n`;
+
+                if (isFlakey) {
+                  flakeSummaries.push(`#### Flake summary for \`${ result.platform }\`, \`${ result.features }\`, \`${ result.toolchain }\`
+
+            \`\`\`shell
+            ${ result.summary }
+            \`\`\``);
+                }
+              }
+
+              if (flakeSummaries.length == 0) {
+                body += '\nNo flakes detected 🎉\n\n'
+              }
+
+              body += table;
+
+              if (flakeSummaries.length > 0) {
+                body += "\n\n";
+                body += flakeSummaries.join('\n\n');
+              }
+            }
+
+            body = `### ${testFlakeAnalysisHeader}
+
+            ${body}`;
+
+            if (existingComment) {
+              github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existingComment.id,
+                body
+              });
+            } else {
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body
+              });
+            }

.github/workflows/run_test_suite.yaml

@@ -56,86 +56,73 @@ jobs:
       - name: 'Run Linter'
         run: cargo clippy --all -- -D warnings
 
-  run-test-suite-windows:
-    runs-on: windows-latest
+  run-rust-test-suite:
+    name: 'Run Rust test suite'
+    strategy:
+      matrix:
+        features: ['test-kubo,headers', 'test-kubo,headers,rocksdb']
+        platform: ['ubuntu-latest', 'windows-latest', 'macos-13']
+        toolchain: ['stable']
+        exclude:
+          - platform: 'windows-latest'
+            features: 'test-kubo,headers,rocksdb'
+    runs-on: ${{ matrix.platform }}
     steps:
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
       - name: 'Setup Rust'
         run: |
           curl -sSf https://sh.rustup.rs | sh -s -- -y
-      - name: 'Install environment packages'
+          rustup toolchain install ${{matrix.toolchain}}
+      - name: 'Install environment packages (Windows)'
+        if: ${{ matrix.platform == 'windows-latest' }}
         run: |
           choco install -y cmake protoc openssl
         shell: sh
-      - name: Install cargo-binstall
-        uses: cargo-bins/cargo-binstall@v1.4.4
-      - name: Install binaries from cargo
-        run: |
-          cargo binstall cargo-nextest --no-confirm
-      - name: 'Install IPFS Kubo'
-        uses: ibnesayeed/setup-ipfs@master
-        with:
-          ipfs_version: v0.17.0
-          run_daemon: true
-      - name: 'Run Rust native target tests'
-        run: cargo nextest run --features test-kubo,headers --retries 5 --color always
-        env:
-          NOOSPHERE_LOG: academic
-
-  run-test-suite-linux:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-      - name: 'Setup Rust'
-        run: |
-          curl -sSf https://sh.rustup.rs | sh -s -- -y
-      - name: 'Install environment packages'
+      - name: 'Install environment packages (Linux)'
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
         run: |
           sudo apt-get update -qqy
           sudo apt-get install jq protobuf-compiler cmake
-      - name: Install cargo-binstall
-        uses: cargo-bins/cargo-binstall@v1.4.4
-      - name: Install binaries from cargo
-        run: |
-          cargo binstall cargo-nextest --no-confirm
       - name: 'Install IPFS Kubo'
         uses: ibnesayeed/setup-ipfs@master
         with:
           ipfs_version: v0.17.0
           run_daemon: true
-      - name: 'Run Rust native target tests'
-        run: cargo nextest run --features test-kubo,headers --retries 5 --color always
-        env:
-          NOOSPHERE_LOG: academic
-
-  run-test-suite-linux-rocksdb:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-      - name: 'Setup Rust'
-        run: |
-          curl -sSf https://sh.rustup.rs | sh -s -- -y
-      - name: 'Install environment packages'
-        run: |
-          sudo apt-get update -qqy
-          sudo apt-get install jq protobuf-compiler cmake libclang-dev
       - name: Install cargo-binstall
         uses: cargo-bins/cargo-binstall@v1.4.4
       - name: Install binaries from cargo
         run: |
-          cargo binstall cargo-nextest --no-confirm
-      - name: 'Install IPFS Kubo'
-        uses: ibnesayeed/setup-ipfs@master
-        with:
-          ipfs_version: v0.17.0
-          run_daemon: true
-      - name: 'Run Rust native target tests (RocksDB)'
-        run: cargo nextest run --features rocksdb,test-kubo,headers --retries 5 --color always
+          cargo +${{ matrix.toolchain }} binstall cargo-nextest --no-confirm --force
+      - name: 'Run Rust tests'
+        shell: bash
+        run: |
+          mkdir -p test-results
+
+          echo "${{ matrix.platform }}
+          ${{ matrix.features }}
+          ${{ matrix.toolchain}}" > test-results/target
+
+          FEATURES="";
+
+          if [ -n "${{matrix.features}}" ]; then
+            FEATURES="--features ${{matrix.features}}"
+          fi
+
+          cargo +${{ matrix.toolchain }} nextest run $FEATURES --retries 5 --color always 2>&1 | tee test-results/log
         env:
           NOOSPHERE_LOG: academic
+      - uses: actions/upload-artifact@v3
+        with:
+          name: test-results-${{ hashFiles('./test-results') }}
+          path: ./test-results
+
+  report-test-flakes:
+    name: 'Report test flakes (Linux)'
+    needs: ['run-rust-test-suite']
+    if: always()
+    uses: ./.github/workflows/report_test_flakes.yaml
+    secrets: inherit
 
   run-test-suite-linux-c:
     runs-on: ubuntu-latest