Merge branch 'master' into oty-nuke-lib-substrate-frame-cli

.github/codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  precision: 2
+  round: down
+  range: "1...100"
+  status:
+    project:
+      default:
+        target: 1.0
+        threshold: 2.0

.github/env

@@ -0,0 +1 @@
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408"

.github/workflows/check-publish.yml

@@ -20,7 +20,7 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish@0.3.0
+        run: cargo install parity-publish@0.5.1
 
       - name: parity-publish check
-        run: parity-publish check --allow-unpublished
+        run: parity-publish --color always check --allow-unpublished

.github/workflows/check-semver.yml

@@ -0,0 +1,55 @@
+name: Check semver
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - prdoc/*.prdoc
+
+jobs:
+  check-semver:
+    runs-on: ubuntu-latest
+    container:
+      image: docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
+        with:
+          cache-on-failure: true
+
+      - name: Rust compilation prerequisites
+        run: |
+          rustup default nightly-2024-03-01
+          rustup target add wasm32-unknown-unknown --toolchain nightly-2024-03-01
+          rustup component add rust-src --toolchain nightly-2024-03-01
+
+      - name: install parity-publish
+        run: cargo install parity-publish@0.5.1
+
+      - name: extra git setup
+        run: |
+          git config --global --add safe.directory '*'
+          git fetch --no-tags --no-recurse-submodules --depth=1 origin master
+          git branch old origin/master
+
+      - name: check semver
+        run: |
+          export CARGO_TARGET_DIR=target
+          export RUSTFLAGS='-A warnings -A missing_docs'
+          if ! parity-publish --color always prdoc --since old --validate prdoc/pr_$PR.prdoc --toolchain nightly-2024-03-01 -v; then
+            cat <<EOF
+          👋 Hello developer! The SemVer information that you declared in the prdoc file did not match what the CI detected.
+
+          Please check the output above and see the following links for more help:
+          - https://github.com/paritytech/polkadot-sdk/blob/master/docs/contributor/prdoc.md#record-semver-changes
+          - https://forum.polkadot.network/t/psa-polkadot-sdk-to-use-semver
+
+          Otherwise feel free to ask in the Merge Request or in Matrix chat.
+          EOF
+
+            exit 1
+          fi
+        env:
+          PR: ${{ github.event.pull_request.number }}

.github/workflows/claim-crates.yml

@@ -18,9 +18,9 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish@0.3.0
+        run: cargo install parity-publish@0.5.1
 
       - name: parity-publish claim
         env:
           PARITY_PUBLISH_CRATESIO_TOKEN: ${{ secrets.CRATESIO_PUBLISH_CLAIM_TOKEN }}
-        run: parity-publish claim
+        run: parity-publish --color always claim

.github/workflows/quick-checks.yml

@@ -0,0 +1,81 @@
+# Checks that doesn't require heavy lifting, like formatting, linting, etc.
+name: quick-checks
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  merge_group:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  set-image:
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: arc-runners-polkadot-sdk-default
+    timeout-minutes: 10
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+  fmt:
+    runs-on: arc-runners-polkadot-sdk-default
+    timeout-minutes: 10
+    needs: [set-image]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Cargo fmt
+        run: cargo +nightly fmt --all -- --check
+  check-dependency-rules:
+    runs-on: arc-runners-polkadot-sdk-default
+    timeout-minutes: 10
+    needs: [set-image]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: check dependency rules
+        run: |
+          cd substrate/
+          ../.gitlab/ensure-deps.sh
+  check-rust-feature-propagation:
+    runs-on: arc-runners-polkadot-sdk-default
+    # runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [set-image]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: run zepter
+        run: zepter run check
+  test-rust-features:
+    runs-on: arc-runners-polkadot-sdk-default
+    # runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [set-image]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: run rust features
+        run: bash .gitlab/rust-features.sh .
+  check-toml-format:
+    runs-on: arc-runners-polkadot-sdk-default
+    timeout-minutes: 10
+    needs: [set-image]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: check toml format
+        run: |
+          taplo format --check --config .config/taplo.toml
+          echo "Please run `taplo format --config .config/taplo.toml` to fix any toml formatting issues"

.github/workflows/release-30_publish_release_draft.yml

@@ -132,6 +132,7 @@ jobs:
   post_to_matrix:
     runs-on: ubuntu-latest
     needs: publish-release-draft
+    environment: release
     strategy:
       matrix:
         channel:

.github/workflows/release-99_notif-published.yml

@@ -48,5 +48,3 @@ jobs:
             Release version: [${{github.event.release.tag_name}}](${{github.event.release.html_url}})
 
             -----
-
-            ${{github.event.release.body}}

.github/workflows/test-github-actions.yml

@@ -8,15 +8,25 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-env:
-  CARGO_NET_GIT_FETCH_WITH_CLI: true
-
 jobs:
+  set-image:
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
   test-linux-stable-int:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
+    needs: [set-image]
     container:
-      image: "docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408"
+      image: ${{ needs.set-image.outputs.IMAGE }}
     env:
       RUSTFLAGS: "-C debug-assertions -D warnings"
       RUST_BACKTRACE: 1
@@ -30,10 +40,11 @@ jobs:
       - name: script
         run: WASM_BUILD_NO_COLOR=1 time cargo test -p staging-node-cli --release --locked -- --ignored
   quick-benchmarks:
-    runs-on: arc-runners-polkadot-sdk
+    runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
+    needs: [set-image]
     container:
-      image: "docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408"
+      image: ${{ needs.set-image.outputs.IMAGE }}
     env:
       RUSTFLAGS: "-C debug-assertions -D warnings"
       RUST_BACKTRACE: "full"
@@ -43,4 +54,4 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
-        run: time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
+        run: time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet

.gitlab/pipeline/test.yml

@@ -5,6 +5,113 @@
     - job: job-starter
       artifacts: false
 
+#
+#
+#
+.codecov-check:
+  script:
+    - >
+      if command -v codecovcli -h >/dev/null 2>&1; then
+        codecovcli --version;
+      else
+        echo "downloading codecovcli";
+        curl -s -o codecovcli https://cli.codecov.io/latest/linux/codecov;
+        chmod +x codecovcli;
+        mv codecovcli /usr/local/bin/codecovcli;
+      fi
+    #
+    - codecovcli --version
+
+#
+#
+#  
+codecov-start:
+  stage: test
+  when: manual
+  allow_failure: false
+  extends:
+    - .kubernetes-env
+    - .common-refs
+    - .pipeline-stopper-artifacts
+    - .run-immediately
+  script:
+    - !reference [.codecov-check, script]
+    - >
+      if [ "$CI_COMMIT_REF_NAME" != "master" ]; then
+        codecovcli -v create-commit -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --fail-on-error --pr ${CI_COMMIT_REF_NAME} --git-service github;
+        codecovcli -v create-report -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --fail-on-error --pr ${CI_COMMIT_REF_NAME} --git-service github;
+      else
+        codecovcli -v create-commit -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --fail-on-error --git-service github;
+        codecovcli -v create-report -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --fail-on-error --git-service github;      
+      fi
+
+#
+#
+#
+codecov-finish:
+  stage: test
+  extends:
+    - .kubernetes-env
+    - .common-refs
+    - .pipeline-stopper-artifacts  
+  needs:
+    - test-linux-stable-codecov
+  script:
+    - !reference [.codecov-check, script]  
+    - codecovcli -v create-report-results -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --git-service github
+    - codecovcli -v get-report-results -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --git-service github
+    - codecovcli -v send-notifications -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --git-service github
+
+#
+#
+#
+test-linux-stable-codecov:
+  stage: test
+  needs:
+    - codecov-start
+  extends:
+    - .docker-env
+    - .common-refs
+    - .pipeline-stopper-artifacts
+  variables:
+    CI_IMAGE: europe-docker.pkg.dev/parity-build/ci-images/ci-unified:bullseye-1.77.0
+    RUST_TOOLCHAIN: stable
+    RUSTFLAGS: "-Cdebug-assertions=y -Cinstrument-coverage"
+    LLVM_PROFILE_FILE: "target/coverage/cargo-test-${CI_NODE_INDEX}-%p-%m.profraw"
+    CARGO_INCREMENTAL: 0    
+    FORKLIFT_BYPASS: "true"  
+  parallel: 2
+  script:
+    # tools
+    - !reference [.codecov-check, script]    
+    - rustup component add llvm-tools-preview
+    - mkdir -p target/coverage/result/  
+    # Place real test call here
+    - >
+      time cargo nextest run -p polkadot \
+        --locked \
+        --release \
+        --no-fail-fast \
+        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
+    # generate and upload reports
+    - >
+      grcov \
+        target/coverage/ \
+        --binary-path ./target/release/ \
+        -s . \
+        -t lcov \
+        --branch \
+        -o target/coverage/result/report-${CI_NODE_INDEX}.lcov
+    - ls -l target/coverage/result/  
+    - >
+      if [ "$CI_COMMIT_REF_NAME" != "master" ]; then  
+        codecovcli -v do-upload -f target/coverage/result/report-${CI_NODE_INDEX}.lcov --disable-search -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --fail-on-error --pr ${CI_COMMIT_REF_NAME} --git-service github;
+      else
+        codecovcli -v do-upload -f target/coverage/result/report-${CI_NODE_INDEX}.lcov --disable-search -t ${CODECOV_TOKEN} -r paritytech/polkadot-sdk --commit-sha ${CI_COMMIT_SHA} --fail-on-error --git-service github;
+      fi
+          
+  #  
+
 test-linux-stable:
   stage: test
   extends: