Merge branch 'master' into gui-fix-5184

.github/actions/cargo-check-runtimes/action.yml

@@ -0,0 +1,22 @@
+name: 'cargo check runtimes'
+description: 'Runs `cargo check` for every directory in provided root.'
+inputs:
+  root:
+    description: "Root directory. Expected to contain several cargo packages inside."
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Check
+      shell: bash
+      run: |
+        mkdir -p ~/.forklift
+        cp .forklift/config.toml ~/.forklift/config.toml
+        cd ${{ inputs.root }}
+        for directory in $(echo */); do
+          echo "_____Running cargo check for ${directory} ______";
+          cd ${directory};
+          pwd;
+          SKIP_WASM_BUILD=1 forklift cargo check --locked;
+          cd ..;
+        done

.github/commands-readme.md

@@ -10,6 +10,7 @@ The current available command actions are:
 
 - [Command FMT](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-fmt.yml)
 - [Command Update UI](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-update-ui.yml)
+- [Command Prdoc](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-prdoc.yml)
 - [Command Sync](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-sync.yml)
 - [Command Bench](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench.yml)
 - [Command Bench All](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench-all.yml)
@@ -235,6 +236,16 @@ You can use the following [`gh cli`](https://cli.github.com/) inside the repo:
 gh workflow run command-bench-overheard.yml -f pr=1000 -f benchmark=substrate -f runtime=rococo -f target_dir=substrate
 ```
 
+### PrDoc
+
+Generate a PrDoc with the crates populated by all modified crates.
+
+Options:
+- `pr`: The PR number to generate the PrDoc for.
+- `audience`: The audience of whom the changes may concern.
+- `bump`: A default bump level for all crates. The PrDoc will likely need to be edited to reflect the actual changes after generation.
+- `overwrite`: Whether to overwrite any existing PrDoc.
+
 ### Sync
 
 Run sync and commit back results to PR.

.github/scripts/generate-prdoc.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+"""
+Generate the PrDoc for a Pull Request with a specific number, audience and bump level.
+
+It downloads and parses the patch from the GitHub API to opulate the prdoc with all modified crates.
+This will delete any prdoc that already exists for the PR if `--force` is passed.
+
+Usage:
+	python generate-prdoc.py --pr 1234 --audience "TODO" --bump "TODO"
+"""
+
+import argparse
+import os
+import re
+import sys
+import subprocess
+import toml
+import yaml
+import requests
+
+from github import Github
+import whatthepatch
+from cargo_workspace import Workspace
+
+# Download the patch and pass the info into `create_prdoc`.
+def from_pr_number(n, audience, bump, force):
+	print(f"Fetching PR '{n}' from GitHub")
+	g = Github()
+
+	repo = g.get_repo("paritytech/polkadot-sdk")
+	pr = repo.get_pull(n)
+
+	patch_url = pr.patch_url
+	patch = requests.get(patch_url).text
+
+	create_prdoc(n, audience, pr.title, pr.body, patch, bump, force)
+
+def create_prdoc(pr, audience, title, description, patch, bump, force):
+	path = f"prdoc/pr_{pr}.prdoc"
+
+	if os.path.exists(path):
+		if force == True:
+			print(f"Overwriting existing PrDoc for PR {pr}")
+		else:
+			print(f"PrDoc already exists for PR {pr}. Use --force to overwrite.")
+			sys.exit(1)
+	else:
+		print(f"No preexisting PrDoc for PR {pr}")
+
+	prdoc = { "doc": [{}], "crates": [] }
+
+	prdoc["title"] = title
+	prdoc["doc"][0]["audience"] = audience
+	prdoc["doc"][0]["description"] = description
+
+	workspace = Workspace.from_path(".")
+
+	modified_paths = []
+	for diff in whatthepatch.parse_patch(patch):
+		modified_paths.append(diff.header.new_path)
+
+	modified_crates = {}
+	for p in modified_paths:
+		# Go up until we find a Cargo.toml
+		p = os.path.join(workspace.path, p)
+		while not os.path.exists(os.path.join(p, "Cargo.toml")):
+			p = os.path.dirname(p)
+
+		with open(os.path.join(p, "Cargo.toml")) as f:
+			manifest = toml.load(f)
+
+		if not "package" in manifest:
+			print(f"File was not in any crate: {p}")
+			continue
+
+		crate_name = manifest["package"]["name"]
+		if workspace.crate_by_name(crate_name).publish:
+			modified_crates[crate_name] = True
+		else:
+			print(f"Skipping unpublished crate: {crate_name}")
+
+	print(f"Modified crates: {modified_crates.keys()}")
+
+	for crate_name in modified_crates.keys():
+		entry = { "name": crate_name }
+
+		if bump == 'silent' or bump == 'ignore' or bump == 'no change':
+			entry["validate"] = False
+		else:
+			entry["bump"] = bump
+
+		print(f"Adding crate {entry}")
+		prdoc["crates"].append(entry)
+
+	# write the parsed PR documentation back to the file
+	with open(path, "w") as f:
+		yaml.dump(prdoc, f)
+
+def parse_args():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("--pr", type=int, required=True)
+	parser.add_argument("--audience", type=str, default="TODO")
+	parser.add_argument("--bump", type=str, default="TODO")
+	parser.add_argument("--force", type=str)
+	return parser.parse_args()
+
+if __name__ == "__main__":
+	args = parse_args()
+	force = True if args.force.lower() == "true" else False
+	print(f"Args: {args}, force: {force}")
+	from_pr_number(args.pr, args.audience, args.bump, force)

.github/workflows/check-cargo-check-runtimes.yml

@@ -0,0 +1,136 @@
+name: Check Cargo Check Runtimes
+
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened, ready_for_review, labeled ]
+
+
+# Jobs in this workflow depend on each other, only for limiting peak amount of spawned workers
+
+jobs:
+  # GitHub Actions allows using 'env' in a container context.
+  # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+  # This workaround sets the container image for each job using 'set-image' job output.
+  set-image:
+    if: contains(github.event.label.name, 'GHA-migration') || contains(github.event.pull_request.labels.*.name, 'GHA-migration')
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+  check-runtime-assets:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    needs: [set-image]
+    timeout-minutes: 20
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run cargo check
+        uses: ./.github/actions/cargo-check-runtimes
+        with:
+          root: cumulus/parachains/runtimes/assets
+
+  check-runtime-collectives:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    needs: [check-runtime-assets, set-image]
+    timeout-minutes: 20
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run cargo check
+        uses: ./.github/actions/cargo-check-runtimes
+        with:
+          root: cumulus/parachains/runtimes/collectives
+
+  check-runtime-coretime:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    needs: [check-runtime-assets, set-image]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run cargo check
+        uses: ./.github/actions/cargo-check-runtimes
+        with:
+          root: cumulus/parachains/runtimes/coretime
+
+  check-runtime-bridge-hubs:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    needs: [set-image]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run cargo check
+        uses: ./.github/actions/cargo-check-runtimes
+        with:
+          root: cumulus/parachains/runtimes/bridge-hubs
+
+  check-runtime-contracts:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    needs: [check-runtime-collectives, set-image]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run cargo check
+        uses: ./.github/actions/cargo-check-runtimes
+        with:
+          root: cumulus/parachains/runtimes/contracts
+
+  check-runtime-starters:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    needs: [check-runtime-assets, set-image]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run cargo check
+        uses: ./.github/actions/cargo-check-runtimes
+        with:
+          root: cumulus/parachains/runtimes/starters
+
+  check-runtime-testing:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    needs: [check-runtime-starters, set-image]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run cargo check
+        uses: ./.github/actions/cargo-check-runtimes
+        with:
+          root: cumulus/parachains/runtimes/testing
+
+  confirm-required-jobs-passed:
+    runs-on: ubuntu-latest
+    name: All check-runtime-* tests passed
+    # If any new job gets added, be sure to add it to this array
+    needs:
+      - check-runtime-assets
+      - check-runtime-collectives
+      - check-runtime-coretime
+      - check-runtime-bridge-hubs
+      - check-runtime-contracts
+      - check-runtime-starters
+      - check-runtime-testing
+    steps:
+      - run: echo '### Good job! All the tests passed 🚀' >> $GITHUB_STEP_SUMMARY

.github/workflows/check-runtime-migration.yml

@@ -6,6 +6,9 @@ on:
       - master
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
+  # Take a snapshot at 5am when most SDK devs are not working.
+  schedule:
+    - cron: '0 5 * * *'
   merge_group:
   workflow_dispatch:
 

.github/workflows/command-prdoc.yml

@@ -0,0 +1,90 @@
+name: Command PrDoc
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr:
+        type: number
+        description: Number of the Pull Request
+        required: true
+      bump:
+        type: choice
+        description: Default bump level for all crates
+        default: "TODO"
+        required: true
+        options:
+          - "TODO"
+          - "no change"
+          - "patch"
+          - "minor"
+          - "major"
+      audience:
+        type: choice
+        description: Audience of the PrDoc
+        default: "TODO"
+        required: true
+        options:
+          - "TODO"
+          - "Runtime Dev"
+          - "Runtime User"
+          - "Node Dev"
+          - "Node User"
+      overwrite:
+        type: choice
+        description: Overwrite existing PrDoc
+        default: "true"
+        required: true
+        options:
+          - "true"
+          - "false"
+
+concurrency:
+  group: command-prdoc
+  cancel-in-progress: true
+
+jobs:
+  set-image:
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+  cmd-prdoc:
+    needs: [set-image]
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Download repo
+        uses: actions/checkout@v4
+      - name: Install gh cli
+        id: gh
+        uses: ./.github/actions/set-up-gh
+        with:
+          pr-number: ${{ inputs.pr }}
+          GH_TOKEN: ${{ github.token }}
+      - name: Generate PrDoc
+        run: |
+          python3 -m pip install -q cargo-workspace PyGithub whatthepatch pyyaml toml
+
+          python3 .github/scripts/generate-prdoc.py --pr "${{ inputs.pr }}" --bump "${{ inputs.bump }}" --audience "${{ inputs.audience }}" --force "${{ inputs.overwrite }}"
+
+      - name: Report failure
+        if: ${{ failure() }}
+        run: gh pr comment ${{ inputs.pr }} --body "<h2>Command failed ❌</h2> Run by @${{ github.actor }} for <code>${{ github.workflow }}</code> failed. See logs <a href=\"$RUN\">here</a>."
+        env:
+          RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          GH_TOKEN: ${{ github.token }}
+      - name: Push Commit
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: Add PrDoc (auto generated)
+          branch: ${{ steps.gh.outputs.branch }}
+          file_pattern: 'prdoc/*.prdoc'