diff --git a/.github/workflows/report_test_flakes.yaml b/.github/workflows/report_test_flakes.yaml new file mode 100644 index 000000000..47817bd59 --- /dev/null +++ b/.github/workflows/report_test_flakes.yaml @@ -0,0 +1,83 @@ +on: + workflow_call: + inputs: + target: + required: true + type: string + +name: 'Workflow Analysis' + +jobs: + report-test-flakes: + name: 'Report test flakes' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/download-artifact@v3 + - name: Parse test results + run: | + RESULTS_PATH="./test-results-${{ inputs.target }}/test-results.log" + + csplit -q "$RESULTS_PATH" %^------------% + + if [[ -f "./xx00" ]]; then + SUMMARY=`tail ./xx00 -n+2` + + if [[ "$SUMMARY" =~ 'FLAKY' ]]; then + # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + { + echo "summary<> $GITHUB_OUTPUT + fi + fi + - name: Report flakes + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const flakeyTestsHeader = 'Test flake analysis: ${{ inputs.target }}'; + const existingComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes(flakeyTestsHeader) + }); + + let body; + + if (`${{ steps['Parse test results'].outputs.summary }}`) { + body = `${flakeyTestsHeader} + + \`\`\`shell + ${{ steps['Parse test results'].outputs.summary }} + \`\`\` + `; + } else { + body = `${flakeyTestsHeader} + + No flakes detected! + `; + } + + if (existingComment) { + github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body + }) + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) + } diff --git a/.github/workflows/run_test_suite.yaml b/.github/workflows/run_test_suite.yaml index 1ffae7db5..fa5254b3e 100644 --- a/.github/workflows/run_test_suite.yaml +++ b/.github/workflows/run_test_suite.yaml @@ -56,86 +56,204 @@ jobs: - name: 'Run Linter' run: cargo clippy --all -- -D warnings - run-test-suite-windows: - runs-on: windows-latest + run-full-test-suite: + strategy: + matrix: + features: ['test-kubo,headers', 'test-kubo,headers,rocksdb'] + platform: ['windows-latest', 'ubuntu-latest'] + toolchain: ['stable', 'nightly'] + runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 - name: 'Setup Rust' run: | curl -sSf https://sh.rustup.rs | sh -s -- -y - - name: 'Install environment packages' + rustup toolchain install ${{matrix.toolchain}} + - name: 'Install environment packages (Windows)' + if: ${{ matrix.platform == 'windows-latest' }} run: | choco install -y cmake protoc openssl shell: sh - - name: Install cargo-binstall - uses: cargo-bins/cargo-binstall@v1.4.4 - - name: Install binaries from cargo - run: | - cargo binstall cargo-nextest --no-confirm - - name: 'Install IPFS Kubo' - uses: ibnesayeed/setup-ipfs@master - with: - ipfs_version: v0.17.0 - run_daemon: true - - name: 'Run Rust native target tests' - run: cargo nextest run --features test-kubo,headers --retries 5 --color always - env: - NOOSPHERE_LOG: academic - - run-test-suite-linux: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: 'Setup Rust' - run: | - curl -sSf https://sh.rustup.rs | sh -s -- -y - - name: 'Install environment packages' + - name: 'Install environment packages (Linux)' + if: ${{ matrix.platform == 'ubuntu-latest' }} run: | sudo apt-get update -qqy sudo apt-get install jq protobuf-compiler cmake - - name: Install cargo-binstall - uses: cargo-bins/cargo-binstall@v1.4.4 - - name: Install binaries from cargo - run: | - cargo binstall cargo-nextest --no-confirm - name: 'Install IPFS Kubo' uses: ibnesayeed/setup-ipfs@master with: ipfs_version: v0.17.0 run_daemon: true - - name: 'Run Rust native target tests' - run: cargo nextest run --features test-kubo,headers --retries 5 --color always - env: - NOOSPHERE_LOG: academic - - run-test-suite-linux-rocksdb: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: 'Setup Rust' - run: | - curl -sSf https://sh.rustup.rs | sh -s -- -y - - name: 'Install environment packages' - run: | - sudo apt-get update -qqy - sudo apt-get install jq protobuf-compiler cmake libclang-dev - name: Install cargo-binstall uses: cargo-bins/cargo-binstall@v1.4.4 - name: Install binaries from cargo run: | - cargo binstall cargo-nextest --no-confirm - - name: 'Install IPFS Kubo' - uses: ibnesayeed/setup-ipfs@master - with: - ipfs_version: v0.17.0 - run_daemon: true - - name: 'Run Rust native target tests (RocksDB)' - run: cargo nextest run --features rocksdb,test-kubo,headers --retries 5 --color always + cargo +${{ matrix.toolchain }} binstall cargo-nextest --no-confirm --force + - name: 'Run Rust tests' + run: cargo +${{ matrix.toolchain }} nextest run --features ${{ matrix.features }} --retries 5 --color always 2>&1 | tee test-results.log env: NOOSPHERE_LOG: academic + - name: Parse test results + run: | + RESULTS_PATH="./test-results.log" + + csplit -q "$RESULTS_PATH" %^------------% + + if [[ -f "./xx00" ]]; then + SUMMARY=`tail ./xx00 -n+2` + + if [[ "$SUMMARY" =~ 'FLAKY' ]]; then + # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + { + echo "summary<> $GITHUB_OUTPUT + fi + fi + - name: Report test flakes + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const flakeyTestsHeader = 'Test flake analysis (`${{ matrix.platform }}`, `${{ matrix.features }}`, `${{ matrix.toolchain }}`)'; + const existingComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes(flakeyTestsHeader) + }); + + let body; + + if (`${{ steps['Parse test results'].outputs.summary }}`) { + body = `#### 🟡 ${flakeyTestsHeader} + + Test flakes detected: + + \`\`\`shell + ${{ steps['Parse test results'].outputs.summary }} + \`\`\` + `; + } else { + body = `#### 🟢 ${flakeyTestsHeader} + + No flakes detected 🎉 + `; + } + + if (existingComment) { + github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body + }) + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) + } + + # run-test-suite-linux: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: Swatinem/rust-cache@v2 + # - name: 'Setup Rust' + # run: | + # curl -sSf https://sh.rustup.rs | sh -s -- -y + # - name: 'Install environment packages' + # run: | + # sudo apt-get update -qqy + # sudo apt-get install jq protobuf-compiler cmake + # - name: Install cargo-binstall + # uses: cargo-bins/cargo-binstall@v1.4.4 + # - name: Install binaries from cargo + # run: | + # cargo binstall cargo-nextest --no-confirm --force + # - name: 'Install IPFS Kubo' + # uses: ibnesayeed/setup-ipfs@master + # with: + # ipfs_version: v0.17.0 + # run_daemon: true + # - name: 'Run Rust native target tests' + # run: cargo nextest run --features test-kubo,headers --retries 5 --color always 2>&1 | tee test-results.log + # env: + # NOOSPHERE_LOG: academic + # - uses: actions/upload-artifact@v3 + # with: + # name: test-results-linux + # path: ./test-results.log + + # report-test-flakes: + # name: 'Report test flakes (Linux)' + # needs: ['run-test-suite-linux'] + # uses: ./.github/workflows/report_test_flakes.yaml + # secrets: inherit + # with: + # target: 'linux' + + # run-test-suite-windows: + # runs-on: windows-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: Swatinem/rust-cache@v2 + # - name: 'Setup Rust' + # run: | + # curl -sSf https://sh.rustup.rs | sh -s -- -y + # - name: 'Install environment packages' + # run: | + # choco install -y cmake protoc openssl + # shell: sh + # - name: Install cargo-binstall + # uses: cargo-bins/cargo-binstall@v1.4.4 + # - name: Install binaries from cargo + # run: | + # cargo binstall cargo-nextest --no-confirm + # - name: 'Install IPFS Kubo' + # uses: ibnesayeed/setup-ipfs@master + # with: + # ipfs_version: v0.17.0 + # run_daemon: true + # - name: 'Run Rust native target tests' + # run: cargo nextest run --features test-kubo,headers --retries 5 --color always 2>&1 | tee test-results.log + # env: + # NOOSPHERE_LOG: academic + + # run-test-suite-linux-rocksdb: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: Swatinem/rust-cache@v2 + # - name: 'Setup Rust' + # run: | + # curl -sSf https://sh.rustup.rs | sh -s -- -y + # - name: 'Install environment packages' + # run: | + # sudo apt-get update -qqy + # sudo apt-get install jq protobuf-compiler cmake libclang-dev + # - name: Install cargo-binstall + # uses: cargo-bins/cargo-binstall@v1.4.4 + # - name: Install binaries from cargo + # run: | + # cargo binstall cargo-nextest --no-confirm --force + # - name: 'Install IPFS Kubo' + # uses: ibnesayeed/setup-ipfs@master + # with: + # ipfs_version: v0.17.0 + # run_daemon: true + # - name: 'Run Rust native target tests (RocksDB)' + # run: cargo nextest run --features rocksdb,test-kubo,headers --retries 5 --color always + # env: + # NOOSPHERE_LOG: academic run-test-suite-linux-c: runs-on: ubuntu-latest