Merge branch 'dev' into GithubAction-DownloadTest

.editorconfig

@@ -10,3 +10,13 @@ indent_style = space
 
 [*.{md,yml,yaml,html,css,scss,js,cff}]
 indent_size = 2
+
+# ignore python and markdown files
+[*.py]
+indent_style = unset
+
+[**/{CONTRIBUTING,README}.md]
+indent_style = unset
+
+[**/Makefile]
+indent_style = unset

.github/.coveragerc

@@ -1,2 +1,5 @@
 [run]
-omit = nf_core/pipeline-template/*
+omit = nf_core/*-template/*
+source = nf_core
+relative_files = True
+

.github/CONTRIBUTING.md

@@ -35,45 +35,28 @@ pip install -e .
 
 ## Code formatting
 
-### Black
+### Ruff
 
-All Python code in nf-core/tools must be passed through the [Black Python code formatter](https://black.readthedocs.io/en/stable/).
+All Python code in nf-core/tools must be passed through the [Ruff code linter and formatter](https://github.com/astral-sh/ruff).
 This ensures a harmonised code formatting style throughout the package, from all contributors.
 
-You can run Black on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository:
+You can run Ruff on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository:
 
 ```bash
-black .
+ruff format .
 ```
 
-Alternatively, Black has [integrations for most common editors](https://black.readthedocs.io/en/stable/editor_integration.html)
+Alternatively, Ruff has [integrations for most common editors](https://github.com/astral-sh/ruff-lsp) and VSCode(https://github.com/astral-sh/ruff-vscode)
 to automatically format code when you hit save.
-You can also set it up to run when you [make a commit](https://black.readthedocs.io/en/stable/version_control_integration.html).
 
 There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
-any code does not adhere to Black formatting.
+any code does not adhere to Ruff formatting.
 
-### isort
-
-All Python code must also be passed through [isort](https://pycqa.github.io/isort/index.html).
-This ensures a harmonised imports throughout the package, from all contributors.
-
-To run isort on the command line recursively on the whole repository you can use:
-
-```bash
-isort .
-```
-
-isort also has [plugins for most common editors](https://github.com/pycqa/isort/wiki/isort-Plugins)
-to automatically format code when you hit save.
-Or [version control integration](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) to set it up to run when you make a commit.
-
-There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
-any code does not adhere to isort formatting.
+Ruff has been adopted for linting and formatting in replacement of Black, isort (for imports) and pyupgrade. It also includes Flake8.
 
 ### pre-commit hooks
 
-This repository comes with [pre-commit](https://pre-commit.com/) hooks for black, isort and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again.
+This repository comes with [pre-commit](https://pre-commit.com/) hooks for ruff and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again.
 
 You can use the pre-commit hooks if you like, but you don't have to. The CI on Github will run the same checks as the tools installed with pre-commit. If the pre-commit checks pass, then the same checks in the CI will pass, too.
 

.github/workflows/branch.yml

@@ -18,7 +18,7 @@ jobs:
       # If the above check failed, post a comment on the PR explaining the failure
       - name: Post PR comment
         if: failure()
-        uses: mshick/add-pr-comment@v1
+        uses: mshick/add-pr-comment@v2
         with:
           message: |
             ## This PR is against the `master` branch :x:

.github/workflows/changelog.py

@@ -0,0 +1,226 @@
+"""
+Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core
+
+To be called by a CI action. Assumes the following environment variables are set:
+PR_TITLE, PR_NUMBER, GITHUB_WORKSPACE.
+
+Adds a line into the CHANGELOG.md:
+* Looks for the section to add the line to, based on the PR title, e.g. `Template:`, `Modules:`.
+* All other change will go under the "### General" section.
+* If an entry for the PR is already added, it will not run.
+
+Other assumptions:
+- CHANGELOG.md has a running section for an ongoing "dev" version
+(i.e. titled "## nf-core vX.Ydev").
+"""
+
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+REPO_URL = "https://github.com/nf-core/tools"
+
+# Assumes the environment is set by the GitHub action.
+pr_title = os.environ["PR_TITLE"]
+pr_number = os.environ["PR_NUMBER"]
+comment = os.environ.get("COMMENT", "")
+workspace_path = Path(os.environ.get("GITHUB_WORKSPACE", ""))
+
+assert pr_title, pr_title
+assert pr_number, pr_number
+
+# Trim the PR number added when GitHub squashes commits, e.g. "Template: Updated (#2026)"
+pr_title = pr_title.removesuffix(f" (#{pr_number})")
+
+changelog_path = workspace_path / "CHANGELOG.md"
+
+if any(
+    line in pr_title.lower()
+    for line in [
+        "skip changelog",
+        "skip change log",
+        "no changelog",
+        "no change log",
+        "bump version",
+    ]
+):
+    print("Skipping changelog update")
+    sys.exit(0)
+
+
+def _run_cmd(cmd):
+    print(cmd)
+    result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise RuntimeError(f"Error executing command: {result.stderr}")
+    return result
+
+
+def _determine_change_type(pr_title) -> str:
+    """
+    Determine the type of the PR: Template, Download, Linting, Modules, Subworkflows, or General
+    Returns a tuple of the section name and the module info.
+    """
+    sections = {
+        "Template": "### Template updates",
+        "Download": "### Download updates",
+        "Linting": "### Linting updates",
+        "Modules": "### Modules",
+        "Subworkflows": "### Subworkflows",
+    }
+    current_section = "### General"
+
+    # Check if the PR in any of the sections.
+    for section, section_header in sections.items():
+        # check if the PR title contains any of the section headers, with some loose matching, e.g. removing plural and suffixes
+        if re.sub(r"s$", "", section.lower().replace("ing", "")) in pr_title.lower():
+            current_section = section_header
+
+    return current_section
+
+
+# Determine the type of the PR: new module, module update, or core update.
+section = _determine_change_type(pr_title)
+
+# Remove section indicator from the PR title.
+pr_title = re.sub(rf"{section}[:\s]*", "", pr_title, flags=re.IGNORECASE)
+
+# Prepare the change log entry.
+pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))"
+
+# Handle manual changelog entries through comments.
+if comment := comment.removeprefix("@nf-core-bot changelog").strip():
+    pr_title = comment
+new_lines = [
+    f"- {pr_title} {pr_link}\n",
+]
+
+# Finally, updating the changelog.
+# Read the current changelog lines. We will print them back as is, except for one new
+# entry, corresponding to this new PR.
+with changelog_path.open("r") as f:
+    orig_lines = f.readlines()
+updated_lines: List[str] = []
+
+
+def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> str:
+    if line.strip().endswith(pr_link):
+        existing_lines = [line]
+        if new_lines and new_lines == existing_lines and same_section:
+            print(f"Found existing identical entry for this pull request #{pr_number} in the same section:")
+            print("".join(existing_lines))
+            sys.exit(0)  # Just leaving the CHANGELOG intact
+        else:
+            print(
+                f"Found existing entry for this pull request #{pr_number}. It will be replaced and/or moved to proper section"
+            )
+            print("".join(existing_lines))
+            for _ in range(len(existing_lines)):
+                try:
+                    line = orig_lines.pop(0)
+                except IndexError:
+                    break
+    return line
+
+
+# Find the next line in the change log that matches the pattern "## MultiQC v.*dev"
+# If it doesn't exist, exist with code 1 (let's assume that a new section is added
+# manually or by CI when a release is pushed).
+# Else, find the next line that matches the `section` variable, and insert a new line
+# under it (we also assume that section headers are added already).
+inside_version_dev = False
+already_added_entry = False
+while orig_lines:
+    line = orig_lines.pop(0)
+
+    # If the line already contains a link to the PR, don't add it again.
+    line = _skip_existing_entry_for_this_pr(line, same_section=False)
+
+    if line.startswith("# ") and not line.strip() == "# nf-core/tools: Changelog":  # Version header, e.g. "# v2.12dev"
+        updated_lines.append(line)
+
+        # Parse version from the line `# v2.12dev` or
+        # `# [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]` ...
+        if not (m := re.match(r".*(v\d+\.\d+(dev)?).*", line)):
+            print(f"Cannot parse version from line {line.strip()}.", file=sys.stderr)
+            sys.exit(1)
+        version = m.group(1)
+
+        if not inside_version_dev:
+            if not version.endswith("dev"):
+                print(
+                    "Can't find a 'dev' version section in the changelog. Make sure "
+                    "it's created, and all the required sections, e.g. `### Template` are created under it .",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            inside_version_dev = True
+        else:
+            if version.endswith("dev"):
+                print(
+                    f"Found another 'dev' version section in the changelog, make"
+                    f"sure to change it to a 'release' stable version tag. "
+                    f"Line: {line.strip()}",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            # We are past the dev version, so just add back the rest of the lines and break.
+            while orig_lines:
+                line = orig_lines.pop(0)
+                line = _skip_existing_entry_for_this_pr(line, same_section=False)
+                if line:
+                    updated_lines.append(line)
+            break
+        continue
+
+    if inside_version_dev and line.lower().startswith(section.lower()):  # Section of interest header
+        if already_added_entry:
+            print(
+                f"Already added new lines into section {section}, is the section duplicated?",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        updated_lines.append(line)
+        # Collecting lines until the next section.
+        section_lines: List[str] = []
+        while True:
+            line = orig_lines.pop(0)
+            if line.startswith("#"):
+                # Found the next section header, so need to put all the lines we collected.
+                updated_lines.append("\n")
+                _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()]
+                updated_lines.extend(_updated_lines)
+                updated_lines.append("\n")
+                if new_lines:
+                    print(f"Updated {changelog_path} section '{section}' with lines:\n" + "".join(new_lines))
+                else:
+                    print(f"Removed existing entry from {changelog_path} section '{section}'")
+                already_added_entry = True
+                # Pushing back the next section header line
+                orig_lines.insert(0, line)
+                break
+            # If the line already contains a link to the PR, don't add it again.
+            line = _skip_existing_entry_for_this_pr(line, same_section=True)
+            section_lines.append(line)
+    else:
+        updated_lines.append(line)
+
+
+def collapse_newlines(lines: List[str]) -> List[str]:
+    updated = []
+    for idx in range(len(lines)):
+        if idx != 0 and not lines[idx].strip() and not lines[idx - 1].strip():
+            continue
+        updated.append(lines[idx])
+    return updated
+
+
+updated_lines = collapse_newlines(updated_lines)
+
+
+# Finally, writing the updated lines back.
+with changelog_path.open("w") as f:
+    f.writelines(updated_lines)

.github/workflows/changelog.yml

@@ -0,0 +1,88 @@
+name: Update CHANGELOG.md
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened]
+
+jobs:
+  update_changelog:
+    runs-on: ubuntu-latest
+    # Run if comment is on a PR with the main repo, and if it contains the magic keywords.
+    # Or run on PR creation, unless asked otherwise in the title.
+    if: |
+      github.repository_owner == 'nf-core' && (
+        github.event_name == 'pull_request_target' ||
+        github.event.issue.pull_request && startsWith(github.event.comment.body, '@nf-core-bot changelog')
+      )
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }}
+
+      # Action runs on the issue comment, so we don't get the PR by default.
+      # Use the GitHub CLI to check out the PR:
+      - name: Checkout Pull Request
+        env:
+          GH_TOKEN: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }}
+        run: |
+          if [[ "${{ github.event_name }}" == "issue_comment" ]]; then
+            PR_NUMBER="${{ github.event.issue.number }}"
+          elif [[ "${{ github.event_name }}" == "pull_request_target" ]]; then
+            PR_NUMBER="${{ github.event.pull_request.number }}"
+          fi
+          gh pr checkout $PR_NUMBER
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyyaml
+
+      - name: Update CHANGELOG.md from the PR title
+        env:
+          COMMENT: ${{ github.event.comment.body }}
+          GH_TOKEN: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }}
+        run: |
+          if [[ "${{ github.event_name }}" == "issue_comment" ]]; then
+            export PR_NUMBER='${{ github.event.issue.number }}'
+            export PR_TITLE='${{ github.event.issue.title }}'
+          elif [[ "${{ github.event_name }}" == "pull_request_target" ]]; then
+            export PR_NUMBER='${{ github.event.pull_request.number }}'
+            export PR_TITLE='${{ github.event.pull_request.title }}'
+          fi
+          python ${GITHUB_WORKSPACE}/.github/workflows/changelog.py
+
+      - name: Check if CHANGELOG.md actually changed
+        run: |
+          git diff --exit-code ${GITHUB_WORKSPACE}/CHANGELOG.md || echo "changed=YES" >> $GITHUB_ENV
+          echo "File changed: ${{ env.changed }}"
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: "pip"
+
+      - name: Install pre-commit
+        run: pip install pre-commit
+
+      - name: Run pre-commit checks
+        if: env.changed == 'YES'
+        run: |
+          pre-commit run --all-files
+
+      - name: Commit and push changes
+        if: env.changed == 'YES'
+        run: |
+          git config user.email "core@nf-co.re"
+          git config user.name "nf-core-bot"
+          git config push.default upstream
+          git add ${GITHUB_WORKSPACE}/CHANGELOG.md
+          git status
+          git commit -m "[automated] Update CHANGELOG.md [no ci]"
+          git push