introduce code coverage analysis in CI (#10180)

Note: I’m removing the preparatory "Build neard" job here. The reason is
twofold:
- I didn’t manage to get coverage output to actually write *.profraw
files without the whole target folder
- Uploading the whole compressed (to 3G) target folder as an artifact
takes 6 minutes for compression and 15 minutes for upload
- Just building, on the commit on which I was, took 2 minutes,
multiplied by 4 it gives 8 minutes, so quite a lot less than the
compress+upload time
- So overall we’re spending less time by just redoing the compilation
work, as opposed to uploading+downloading it; and the tests themselves
are relatively fast too so having them on the bigger machines (8cores,
enough for them to not be the bottleneck) shouldn’t hog them for too
long.

.github/workflows/ci.yml

@@ -9,24 +9,6 @@ on:
   merge_group:
 
 jobs:
-  build_binary:
-    name: "Build neard"
-    runs-on: ubuntu-22.04-16core
-    steps:
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
-        with:
-          prefix-key: "0" # change this to invalidate CI cache
-          shared-key: "cargo_nextest-linux"
-          save-if: "false" # use the cache from nextest, but don’t double-save
-      - run: cargo build --locked --profile dev-release -p neard --bin neard
-      - uses: actions/upload-artifact@v3
-        with:
-          name: neard
-          path: target/dev-release/neard
-          if-no-files-found: error
-          retention-days: 1
-
   cargo_nextest:
     name: "Cargo Nextest (${{matrix.name}})"
     runs-on: ${{ matrix.os }}
@@ -35,39 +17,79 @@ jobs:
       matrix:
         include:
           - name: Linux
+            id: linux
             cache_id: linux
             os: ubuntu-22.04-16core
             flags: ""
+            run_integ_tests: true
           - name: Linux Nightly
+            id: linux-nightly
             cache_id: linux
             os: ubuntu-22.04-16core
             flags: "--features nightly,test_features"
+            run_integ_tests: true
           - name: MacOS
+            id: macos
             cache_id: macos
             os: macos-latest-xlarge
             # FIXME: some of these tests don't work very well on MacOS at the moment. Should fix
             # them at earliest convenience :)
-            flags: "--exclude integration-tests --exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse"
+            flags: "--exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse"
+            run_integ_tests: false
     timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4
+
+      # Install all the required tools
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-nextest
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-deny
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
+
+      # Setup the dependency rust cache and llvm-cov
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
-      - run: cargo nextest run --locked --workspace -p '*' --cargo-profile dev-release --profile ci ${{ matrix.flags }}
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+
+      # Run unit tests
+      - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile dev-release --profile ci ${{ matrix.flags }}
         env:
           RUST_BACKTRACE: short
+      - run: cargo llvm-cov report --profile dev-release --codecov --output-path unittests.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: unittests.json
+          fail_ci_if_error: true
+          flags: unittests,${{ matrix.id }}
+      # See https://github.com/taiki-e/cargo-llvm-cov/issues/292
+      - run: find target -name '*.profraw' -delete
+
+      # Run integration tests
+      - run: cargo nextest run --locked --package integration-tests --cargo-profile dev-release --profile ci ${{ matrix.flags }}
+        if: matrix.run_integ_tests
+        env:
+          RUST_BACKTRACE: short
+      - run: cargo llvm-cov report --profile dev-release --codecov --output-path integration-tests.json
+        if: matrix.run_integ_tests
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        if: matrix.run_integ_tests
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: integration-tests.json
+          fail_ci_if_error: true
+          flags: integration-tests,${{ matrix.id }}
 
   protobuf_backward_compat:
     name: "Protobuf Backward Compatibility"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: bufbuild/buf-setup-action@1158f4fa81bc02e1ff62abcca6d516c9e24c77da
@@ -77,48 +99,64 @@ jobs:
 
   py_backward_compat:
     name: "Backward Compatibility"
-    needs: build_binary
     runs-on: ubuntu-22.04
-    defaults:
-      run:
-        working-directory: ./pytest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: 3.11
           cache: pip
-      - uses: actions/download-artifact@v3
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
+        with:
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
+      - run: pip3 install --user -r pytest/requirements.txt
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: cargo build --locked --profile dev-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV"
+      - run: cd pytest && python3 tests/sanity/backward_compatible.py
+      - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-backcomp.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
-      - run: pip3 install --user -r requirements.txt
-      - run: python3 tests/sanity/backward_compatible.py
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: pytest-backcomp.json
+          fail_ci_if_error: true
+          flags: pytests,backward-compatibility,linux
 
   py_db_migration:
     name: "Database Migration"
-    needs: build_binary
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        working-directory: ./pytest
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: 3.11
           cache: pip
-      - uses: actions/download-artifact@v3
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
+          crate: cargo-llvm-cov
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
+        with:
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
+      - run: pip3 install --user -r pytest/requirements.txt
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: cargo build --locked --profile dev-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV"
       - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
-      - run: pip3 install --user -r requirements.txt
-      - run: python3 tests/sanity/db_migration.py
+      - run: cd pytest && python3 tests/sanity/db_migration.py
+      - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-dbmigr.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: pytest-dbmigr.json
+          fail_ci_if_error: true
+          flags: pytests,db-migration,linux
 
   py_sanity_checks:
     name: "Sanity Checks"
@@ -132,6 +170,9 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
@@ -140,32 +181,51 @@ jobs:
       - run: pip3 install --user -r pytest/requirements.txt
       # This is the only job that uses `--features nightly` so we build this in-line instead of a
       # separate job like done with the regular neard.
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: cargo build --profile dev-release -p neard --bin neard --features nightly
       # Note: We're not running spin_up_cluster.py for non-nightly
       # because spinning up non-nightly clusters is already covered
       # by other steps in the CI, e.g. upgradable.
       - run: python3 pytest/tests/sanity/spin_up_cluster.py
         env:
           NEAR_ROOT: "target/dev-release"
+      - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-sanity.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: pytest-sanity.json
+          fail_ci_if_error: true
+          flags: pytests,sanity-checks,linux-nightly
 
   py_genesis_check:
     name: "Genesis Changes"
-    needs: build_binary
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: 3.11
           cache: pip
-      - uses: actions/download-artifact@v3
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
-          name: neard
-          path: target/dev-release
-      - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
+          crate: cargo-llvm-cov
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
+        with:
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
       - run: pip3 install --user -r pytest/requirements.txt
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: cargo build --locked --profile dev-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV"
       - run: python3 scripts/state/update_res.py check
+      - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-genesischk.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: pytest-genesischk.json
+          fail_ci_if_error: true
+          flags: pytests,genesis-check,linux
 
   py_style_check:
     name: "Style"
@@ -184,25 +244,33 @@ jobs:
 
   py_upgradability:
     name: "Upgradability"
-    needs: build_binary
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        working-directory: ./pytest
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: 3.11
           cache: pip
-      - uses: actions/download-artifact@v3
-        with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
-      - run: pip3 install --user -r requirements.txt
-      - run: python3 tests/sanity/upgradable.py
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
+        with:
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
+      - run: pip3 install --user -r pytest/requirements.txt
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: cargo build --locked --profile dev-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV"
+      - run: cd pytest && python3 tests/sanity/upgradable.py
+      - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-upgradability.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: pytest-upgradability.json
+          fail_ci_if_error: true
+          flags: pytests,upgradability,linux
 
   rpc_error_schema:
     name: "RPC Schema"