From 5787750d68d0be2c3e7f1744ae42cb1beb661c6b Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 17:50:35 +0000 Subject: [PATCH 01/16] Update nf-test CI so each test is run only once and as its own test --- .github/python/find_changed_files.py | 192 +++++++++++++++++++++++++++ .github/workflows/ci.yml | 118 +++++++++++----- 2 files changed, 278 insertions(+), 32 deletions(-) create mode 100644 .github/python/find_changed_files.py diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py new file mode 100644 index 0000000..87e45bc --- /dev/null +++ b/.github/python/find_changed_files.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python + +# This script is used to identify *.nf.test files for changed functions/processs/workflows/pipelines and *.nf-test files +# with changed dependencies, then return as a JSON list + +import argparse +import json +import logging +import re + +from itertools import chain +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + """ + Parse command line arguments and return an ArgumentParser object. + + Returns: + argparse.ArgumentParser: The ArgumentParser object with the parsed arguments. + """ + parser = argparse.ArgumentParser( + description="Scan *.nf.test files for function/process/workflow name and return as a JSON list" + ) + parser.add_argument( + "-p", + "--paths", + nargs="+", + default=["."], + help="List of directories or files to scan", + ) + parser.add_argument( + "-l", + "--log-level", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + default="INFO", + help="Logging level", + ) + parser.add_argument( + "-t", + "--types", + nargs="+", + choices=["function", "process", "workflow", "pipeline"], + default=["function", "process", "workflow", "pipeline"], + help="Types of tests to include.", + ) + return parser.parse_args() + + +def find_files(paths: list[str]) -> list[Path]: + """ + Find all files matching pattern *.nf.test recursively from a list of paths. + + Args: + paths (list): List of directories or files to scan. + + Returns: + list: List of files matching the pattern *.nf.test. + """ + # this is a bit clunky + result = [] + for path in paths: + path_obj = Path(path) + # If Path is the exact nf-test file add to list: + if path_obj.match("*.nf.test"): + result.append(path_obj) + # Else recursively search for nf-test files: + else: + for file in path_obj.rglob("*.nf.test"): + result.append(file) + return result + + +def process_files(files: list[Path]) -> list[str]: + """ + Process the files and return lines that begin with 'workflow', 'process', or 'function' and have a single string afterwards. + + Args: + files (list): List of files to process. + + Returns: + list: List of lines that match the criteria. + """ + result = [] + for file in files: + with open(file, "r") as f: + is_pipeline_test = True + lines = f.readlines() + for line in lines: + line = line.strip() + if line.startswith(("workflow", "process", "function")): + words = line.split() + if len(words) == 2 and re.match(r'^".*"$', words[1]): + result.append(line) + is_pipeline_test = False + + # If no results included workflow, process or function + # Add a dummy result to fill the 'pipeline' category + if is_pipeline_test: + result.append("pipeline 'PIPELINE'") + + return result + + +def generate( + lines: list[str], types: list[str] = ["function", "process", "workflow", "pipeline"] +) -> dict[str, list[str]]: + """ + Generate a dictionary of function, process and workflow lists from the lines. + + Args: + lines (list): List of lines to process. + types (list): List of types to include. + + Returns: + dict: Dictionary with function, process and workflow lists. + """ + result: dict[str, list[str]] = { + "function": [], + "process": [], + "workflow": [], + "pipeline": [], + } + for line in lines: + words = line.split() + if len(words) == 2: + keyword = words[0] + name = words[1].strip("'\"") # Strip both single and double quotes + if keyword in types: + result[keyword].append(name) + return result + + +def find_changed_dependencies(paths: list[str], tags: list[str]) -> list[Path]: + """ + Find all *.nf.test files with changed dependencies from a list of paths. + + Args: + paths (list): List of directories or files to scan. + tags: List of tags identified as having changes. + + Returns: + list: List of *.nf.test files with changed dependencies. + """ + # this is a bit clunky + result = [] + for path in paths: + path_obj = Path(path) + # find all *.nf-test files + nf_test_files = [] + for file in path_obj.rglob("*.nf.test"): + nf_test_files.append(file) + # find nf-test files with changed dependencies + for nf_test_file in nf_test_files: + with open(nf_test_file, "r") as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if line.startswith("tag"): + words = line.split() + if len(words) == 2 and re.match(r'^".*"$', words[1]): + name = words[1].strip("'\"") # Strip both single and double quotes + if name in tags: + result.append(nf_test_file) + + return list(set(result)) + +if __name__ == "__main__": + + # Utility stuff + # args = parse_args() + logging.basicConfig(level="INFO") + + # Parse nf-test files for target test tags + files = find_files(".") + lines = process_files(files) + result = generate(lines) + + # Get only relevant results (specified by -t) + # Unique using a set + target_results = list( + {item for sublist in map(result.get, ["function", "process", "workflow", "pipeline"]) for item in sublist} + ) + + # Parse files to identify nf-tests with changed dependencies + changed_dep_files = find_changed_dependencies(".", target_results) + + # Combine target nf-test files and nf-test files with changed dependencies + all_nf_tests = list(set(changed_dep_files + files)) + + # Print to stdout + print(json.dumps(all_nf_tests)) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14ad077..77abe26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: env: NXF_ANSI_LOG: false - NFT_VER: "0.8.3" + NFT_VER: "0.8.4" NFT_WORKDIR: "~" NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" @@ -27,44 +27,80 @@ jobs: name: Check for changes runs-on: ubuntu-latest outputs: - # Expose matched filters as job 'tags' output variable - tags: ${{ steps.filter.outputs.changes }} + changes: ${{ steps.changed_files.outputs.any_modified }} + nf_test_files: ${{ steps.list.outputs.nf_test_files }} steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: "x64" + - uses: actions/checkout@v3 - - name: Combine all tags.yml files - id: get_username - run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml - - name: debug - run: cat .github/tags.yml - - uses: dorny/paths-filter@v2 - id: filter with: - filters: ".github/tags.yml" + fetch-depth: 0 - define_nxf_versions: - name: Choose nextflow versions to test against depending on target branch - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.nxf_versions.outputs.matrix }} - steps: - - id: nxf_versions + - uses: tj-actions/changed-files@v42 + id: changed_files + with: + dir_names: "true" + output_renamed_files_as_deleted_and_added: "true" + # Define list of additional rules for testing paths + # Mostly, we define additional 'pipeline' or 'all' tests here + files_yaml: | + ".": + - .github/workflows/** + - nf-test.config + - nextflow.config + tests: + - assets/* + - bin/* + - conf/* + - main.nf + - nextflow_schema.json + + files_ignore: | + .git* + .gitpod.yml + .prettierignore + .prettierrc.yml + **.md + **.png + modules.json + pyproject.toml + tower.yml + + - name: debug + run: | + echo ${{ steps.changed_files.outputs.any_modified }} + echo ${{ steps.changed_files.outputs.all_changed_files }} + echo ${{ steps.changed_files.outputs.changed_keys }} + + - name: nf-test list nf_test_files + id: list + if: ${{ steps.changed_files.outputs.any_modified }} + run: | + echo nf_test_files=$(python \ + .github/python/find_changed_files.py \ + -t pipeline workflow process \ + -p ${{ steps.changed_files.outputs.all_changed_files }} ${{ steps.changed_files.outputs.changed_keys }} \ + ) >> $GITHUB_OUTPUT + + - name: debug2 run: | - if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then - echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT - else - echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT - fi + echo ${{ steps.list.outputs.nf_test_files }} test: - name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} - needs: [changes, define_nxf_versions] - if: needs.changes.outputs.tags != '[]' + name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} + needs: [changes] + if: needs.changes.outputs.changes runs-on: ubuntu-latest strategy: fail-fast: false matrix: - NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }} - tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] + NXF_VER: + - "latest-everything" + - "23.04" + nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"] profile: - "docker" @@ -77,9 +113,6 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Disk space cleanup - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - uses: actions/setup-python@v4 with: python-version: "3.11" @@ -105,9 +138,12 @@ jobs: wget -qO- https://code.askimed.com/install/nf-test | bash sudo mv nf-test /usr/local/bin/ + - name: Disk space cleanup + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: @@ -125,3 +161,21 @@ jobs: if: always() # always run even if the previous step fails with: report_paths: test.xml + + confirm-pass: + runs-on: ubuntu-latest + needs: [test] + if: always() + steps: + - name: All tests ok + if: ${{ !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" From 3e9c4c4e5656b17aada9557a0cd478f1548f8ef7 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 17:53:17 +0000 Subject: [PATCH 02/16] Updated checkv/downloaddatabase module --- modules.json | 2 +- .../checkv/downloaddatabase/environment.yml | 7 ++++ .../nf-core/checkv/downloaddatabase/main.nf | 28 +++++++++++++-- .../nf-core/checkv/downloaddatabase/meta.yml | 8 ++--- .../downloaddatabase/tests/main.nf.test | 35 +++++++++++++++++++ .../downloaddatabase/tests/main.nf.test.snap | 18 ++++++++++ .../downloaddatabase/tests/nextflow.config | 2 ++ .../checkv/downloaddatabase/tests/tags.yml | 2 ++ 8 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 modules/nf-core/checkv/downloaddatabase/environment.yml create mode 100644 modules/nf-core/checkv/downloaddatabase/tests/main.nf.test create mode 100644 modules/nf-core/checkv/downloaddatabase/tests/main.nf.test.snap create mode 100644 modules/nf-core/checkv/downloaddatabase/tests/nextflow.config create mode 100644 modules/nf-core/checkv/downloaddatabase/tests/tags.yml diff --git a/modules.json b/modules.json index f5fa850..5db56ea 100644 --- a/modules.json +++ b/modules.json @@ -37,7 +37,7 @@ }, "checkv/downloaddatabase": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "56cf5a89c2646ad50195635cf18df4028a6928ce", "installed_by": ["modules"] }, "checkv/endtoend": { diff --git a/modules/nf-core/checkv/downloaddatabase/environment.yml b/modules/nf-core/checkv/downloaddatabase/environment.yml new file mode 100644 index 0000000..9dfdcd1 --- /dev/null +++ b/modules/nf-core/checkv/downloaddatabase/environment.yml @@ -0,0 +1,7 @@ +name: checkv_downloaddatabase +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::checkv=1.0.1 diff --git a/modules/nf-core/checkv/downloaddatabase/main.nf b/modules/nf-core/checkv/downloaddatabase/main.nf index bc5086f..8e0bf86 100644 --- a/modules/nf-core/checkv/downloaddatabase/main.nf +++ b/modules/nf-core/checkv/downloaddatabase/main.nf @@ -1,7 +1,7 @@ process CHECKV_DOWNLOADDATABASE { label 'process_low' - conda "bioconda::checkv=1.0.1" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/checkv:1.0.1--pyhdfd78af_0': 'biocontainers/checkv:1.0.1--pyhdfd78af_0' }" @@ -20,7 +20,31 @@ process CHECKV_DOWNLOADDATABASE { """ checkv download_database \\ $args \\ - ./$prefix/ \\ + ./$prefix/ + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + checkv: \$(checkv -h 2>&1 | sed -n 's/^.*CheckV v//; s/: assessing.*//; 1p') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "checkv_db" + + """ + mkdir ${prefix} + touch ${prefix}/README.txt + mkdir ${prefix}/genome_db + touch ${prefix}/genome_db/changelog.tsv + touch ${prefix}/genome_db/checkv_error.tsv + touch ${prefix}/genome_db/checkv_info.tsv + touch ${prefix}/genome_db/checkv_reps.faa + touch ${prefix}/genome_db/checkv_reps.fna + touch ${prefix}/genome_db/checkv_reps.tsv + mkdir ${prefix}/hmm_db + touch ${prefix}/hmm_db/checkv_hmms.tsv + touch ${prefix}/hmm_db/genome_lengths.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/checkv/downloaddatabase/meta.yml b/modules/nf-core/checkv/downloaddatabase/meta.yml index 739a1a9..1ca5415 100644 --- a/modules/nf-core/checkv/downloaddatabase/meta.yml +++ b/modules/nf-core/checkv/downloaddatabase/meta.yml @@ -1,5 +1,4 @@ name: "checkv_downloaddatabase" - description: Construct the database necessary for checkv's quality assessment keywords: - checkv @@ -20,18 +19,17 @@ tools: documentation: https://bitbucket.org/berkeleylab/checkv/src/master/ tool_dev_url: https://bitbucket.org/berkeleylab/checkv/src/master/ doi: "10.1038/s41587-020-00774-7" - licence: "['BSD License']" - + licence: ["BSD License"] output: - versions: type: file description: File containing software versions pattern: "versions.yml" - - checkv_db: type: directory description: directory pointing to database pattern: "${prefix}/" - authors: - "@Joon-Klaps" +maintainers: + - "@Joon-Klaps" diff --git a/modules/nf-core/checkv/downloaddatabase/tests/main.nf.test b/modules/nf-core/checkv/downloaddatabase/tests/main.nf.test new file mode 100644 index 0000000..95f8923 --- /dev/null +++ b/modules/nf-core/checkv/downloaddatabase/tests/main.nf.test @@ -0,0 +1,35 @@ +nextflow_process { + + name "Test Process CHECKV_DOWNLOADDATABASE" + script "../main.nf" + process "CHECKV_DOWNLOADDATABASE" + config "./nextflow.config" + + tag "modules" + tag "modules_nfcore" + tag "checkv" + tag "checkv/downloaddatabase" + + test("No input: download only") { + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.versions).match() } + ) + } + } + + test("No input: download only - stub") { + + options "-stub" + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.versions).match() } + ) + } + } + +} diff --git a/modules/nf-core/checkv/downloaddatabase/tests/main.nf.test.snap b/modules/nf-core/checkv/downloaddatabase/tests/main.nf.test.snap new file mode 100644 index 0000000..6644657 --- /dev/null +++ b/modules/nf-core/checkv/downloaddatabase/tests/main.nf.test.snap @@ -0,0 +1,18 @@ +{ + "No input: download only - stub": { + "content": [ + [ + "versions.yml:md5,29c4fbe7a6cb8b6c0913363d718cce9f" + ] + ], + "timestamp": "2024-01-17T16:40:56.505431212" + }, + "No input: download only": { + "content": [ + [ + "versions.yml:md5,29c4fbe7a6cb8b6c0913363d718cce9f" + ] + ], + "timestamp": "2023-12-11T16:24:08.049483097" + } +} \ No newline at end of file diff --git a/modules/nf-core/checkv/downloaddatabase/tests/nextflow.config b/modules/nf-core/checkv/downloaddatabase/tests/nextflow.config new file mode 100644 index 0000000..3ae5aa1 --- /dev/null +++ b/modules/nf-core/checkv/downloaddatabase/tests/nextflow.config @@ -0,0 +1,2 @@ +docker.fixOwnership = false +docker.runOptions = '-u $(id -u):$(id -g)' diff --git a/modules/nf-core/checkv/downloaddatabase/tests/tags.yml b/modules/nf-core/checkv/downloaddatabase/tests/tags.yml new file mode 100644 index 0000000..23d750a --- /dev/null +++ b/modules/nf-core/checkv/downloaddatabase/tests/tags.yml @@ -0,0 +1,2 @@ +checkv/downloaddatabase: + - "modules/nf-core/checkv/downloaddatabase/**" From aea060c7b9f1bd768aff3a6ff695d370effdf77c Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 18:00:09 +0000 Subject: [PATCH 03/16] Output nf-test file paths as string --- .github/python/find_changed_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index 87e45bc..3b5db42 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -63,11 +63,11 @@ def find_files(paths: list[str]) -> list[Path]: path_obj = Path(path) # If Path is the exact nf-test file add to list: if path_obj.match("*.nf.test"): - result.append(path_obj) + result.append(str(path_obj)) # Else recursively search for nf-test files: else: for file in path_obj.rglob("*.nf.test"): - result.append(file) + result.append(str(file)) return result @@ -161,7 +161,7 @@ def find_changed_dependencies(paths: list[str], tags: list[str]) -> list[Path]: if len(words) == 2 and re.match(r'^".*"$', words[1]): name = words[1].strip("'\"") # Strip both single and double quotes if name in tags: - result.append(nf_test_file) + result.append(str(nf_test_file)) return list(set(result)) From fda8755203ca1e3e968caeda836717d9bf763ed1 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 18:03:29 +0000 Subject: [PATCH 04/16] Updated python script to take only changed files --- .github/python/find_changed_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index 3b5db42..29090c6 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -169,17 +169,17 @@ def find_changed_dependencies(paths: list[str], tags: list[str]) -> list[Path]: # Utility stuff # args = parse_args() - logging.basicConfig(level="INFO") + logging.basicConfig(level=args.log_level) # Parse nf-test files for target test tags - files = find_files(".") + files = find_files(args.paths) lines = process_files(files) result = generate(lines) # Get only relevant results (specified by -t) # Unique using a set target_results = list( - {item for sublist in map(result.get, ["function", "process", "workflow", "pipeline"]) for item in sublist} + {item for sublist in map(result.get, args.types) for item in sublist} ) # Parse files to identify nf-tests with changed dependencies From 9ff858a959587ecdd2fa916b171ed196d90f3eaa Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 18:04:41 +0000 Subject: [PATCH 05/16] Added parse_args to __main__ --- .github/python/find_changed_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index 29090c6..3c77398 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -168,7 +168,7 @@ def find_changed_dependencies(paths: list[str], tags: list[str]) -> list[Path]: if __name__ == "__main__": # Utility stuff - # args = parse_args() + args = parse_args() logging.basicConfig(level=args.log_level) # Parse nf-test files for target test tags From 8d6a3e4ba593b4890d7cb4f34752a7e59cefbb87 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 20:48:40 +0000 Subject: [PATCH 06/16] Test files_yaml job --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77abe26..c16e98c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,6 @@ jobs: # Mostly, we define additional 'pipeline' or 'all' tests here files_yaml: | ".": - - .github/workflows/** - nf-test.config - nextflow.config tests: From fe4c7266f4b99897e9d4f2a97a27fe9e86009db6 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 22:51:08 +0000 Subject: [PATCH 07/16] Fixed nf-test command to run with file instead of tag --- .github/workflows/ci.yml | 2 +- .../anicluster/anicalc/tests/main.nf.test | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c16e98c..1222340 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,7 +142,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test ${{ matrix.nf_test_files }} --verbose --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/modules/local/anicluster/anicalc/tests/main.nf.test b/modules/local/anicluster/anicalc/tests/main.nf.test index 3b967e1..bbba44c 100644 --- a/modules/local/anicluster/anicalc/tests/main.nf.test +++ b/modules/local/anicluster/anicalc/tests/main.nf.test @@ -3,13 +3,8 @@ nextflow_process { name "Test process: ANICLUSTER_ANICALC" script "../main.nf" process "ANICLUSTER_ANICALC" - tag "modules" - tag "modules_local" - tag "anicluster" - tag "anicluster_anicalc" - - test("['modules_local']['blast_txt']") { + test("blast.txt") { when { params { @@ -17,7 +12,10 @@ nextflow_process { } process { """ - input[0] = [ [ id: 'test' ], [ file(params.test_data['modules_local']['blast_txt'], checkIfExists: true) ] ] + input[0] = [ + [ id: 'test' ] + file(params.pipelines_testdata_base_path + 'modules/local/anicluster/anicalc/blast.txt', checkIfExists: true) + ] """ } } @@ -30,7 +28,7 @@ nextflow_process { } } - test("['modules_local']['blast_txt'] - stub") { + test("blast.txt - stub") { options "-stub" @@ -40,7 +38,10 @@ nextflow_process { } process { """ - input[0] = [ [ id: 'test' ], [ file(params.test_data['modules_local']['blast_txt'], checkIfExists: true) ] ] + input[0] = [ + [ id: 'test' ] + file(params.pipelines_testdata_base_path + 'modules/local/anicluster/anicalc/blast.txt', checkIfExists: true) + ] """ } } From 2799f5564c506b769ae17b40b3436833c7d372d8 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 22 Feb 2024 23:11:00 +0000 Subject: [PATCH 08/16] Reverted test data source --- modules/local/anicluster/anicalc/tests/main.nf.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/local/anicluster/anicalc/tests/main.nf.test b/modules/local/anicluster/anicalc/tests/main.nf.test index bbba44c..c60ae12 100644 --- a/modules/local/anicluster/anicalc/tests/main.nf.test +++ b/modules/local/anicluster/anicalc/tests/main.nf.test @@ -13,8 +13,8 @@ nextflow_process { process { """ input[0] = [ - [ id: 'test' ] - file(params.pipelines_testdata_base_path + 'modules/local/anicluster/anicalc/blast.txt', checkIfExists: true) + [ id: 'test' ], + file(params.test_data['modules_local']['blast_txt'], checkIfExists: true) ] """ } @@ -39,8 +39,8 @@ nextflow_process { process { """ input[0] = [ - [ id: 'test' ] - file(params.pipelines_testdata_base_path + 'modules/local/anicluster/anicalc/blast.txt', checkIfExists: true) + [ id: 'test' ], + file(params.test_data['modules_local']['blast_txt'], checkIfExists: true) ] """ } From d2588f5cb88764279210b6a11482289d3908b7f8 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 27 Feb 2024 04:09:35 +0000 Subject: [PATCH 09/16] Fixed nested loop --- .github/python/find_changed_files.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index 3c77398..ec9444c 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -150,18 +150,18 @@ def find_changed_dependencies(paths: list[str], tags: list[str]) -> list[Path]: nf_test_files = [] for file in path_obj.rglob("*.nf.test"): nf_test_files.append(file) - # find nf-test files with changed dependencies - for nf_test_file in nf_test_files: - with open(nf_test_file, "r") as f: - lines = f.readlines() - for line in lines: - line = line.strip() - if line.startswith("tag"): - words = line.split() - if len(words) == 2 and re.match(r'^".*"$', words[1]): - name = words[1].strip("'\"") # Strip both single and double quotes - if name in tags: - result.append(str(nf_test_file)) + # find nf-test files with changed dependencies + for nf_test_file in nf_test_files: + with open(nf_test_file, "r") as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if line.startswith("tag"): + words = line.split() + if len(words) == 2 and re.match(r'^".*"$', words[1]): + name = words[1].strip("'\"") # Strip both single and double quotes + if name in tags: + result.append(str(nf_test_file)) return list(set(result)) From d7a1a712d6dd5371a9364ec2eef7db698268f970 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 27 Feb 2024 23:57:01 +0000 Subject: [PATCH 10/16] Removed changed_files action and replaced it with Python script --- .github/python/find_changed_files.py | 59 +++++++++++++++++++++++----- .github/workflows/ci.yml | 40 +------------------ 2 files changed, 52 insertions(+), 47 deletions(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index ec9444c..a96fc7c 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -10,6 +10,7 @@ from itertools import chain from pathlib import Path +from git import Repo def parse_args() -> argparse.Namespace: @@ -23,11 +24,31 @@ def parse_args() -> argparse.Namespace: description="Scan *.nf.test files for function/process/workflow name and return as a JSON list" ) parser.add_argument( - "-p", - "--paths", + "-r", + "--head_ref", + required=True, + help="Head reference branch (Source branch for a PR).", + ) + parser.add_argument( + "-b", + "--base_ref", + required=True, + help="Base reference branch (Target branch for a PR).", + ) + parser.add_argument( + "-i", + "--ignored_files", nargs="+", - default=["."], - help="List of directories or files to scan", + default=[".git", + ".gitpod.yml", + ".prettierignore", + ".prettierrc.yml", + ".md", + ".png", + "modules.json", + "pyproject.toml", + "tower.yml"], + help="List of files or file substrings to ignore.", ) parser.add_argument( "-l", @@ -47,19 +68,39 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def find_files(paths: list[str]) -> list[Path]: +def find_files(branch1: str, branch2: str, ignore: list[str]) -> list[Path]: """ - Find all files matching pattern *.nf.test recursively from a list of paths. + Find all *.nf.tests that are associated with files that have been changed between two specified branches. Args: - paths (list): List of directories or files to scan. + branch1 (str) : The first branch being compared + branch2 (str) : The second branch being compared + ignore (list) : List of files or file substrings to ignore. Returns: list: List of files matching the pattern *.nf.test. """ + # create repo + repo = Repo(".") + # identify commit on branch1 + branch1_commit = repo.commit(branch1) + # identify commit on branch2 + branch2_commit = repo.commit(branch2) + # compare two branches + diff_index = branch1_commit.diff(branch2_commit) + # collect changed files + changed_files = [] + for file in diff_index: + changed_files.append(file.a_path) + # remove ignored files + for file in changed_files: + for ignored_substring in ignore: + if ignored_substring in file: + changed_files.remove(file) + # this is a bit clunky result = [] - for path in paths: + for path in changed_files: path_obj = Path(path) # If Path is the exact nf-test file add to list: if path_obj.match("*.nf.test"): @@ -172,7 +213,7 @@ def find_changed_dependencies(paths: list[str], tags: list[str]) -> list[Path]: logging.basicConfig(level=args.log_level) # Parse nf-test files for target test tags - files = find_files(args.paths) + files = find_files(args.head_ref, args.base_ref, args.ignored_files) lines = process_files(files) result = generate(lines) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1222340..fabb61a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ jobs: name: Check for changes runs-on: ubuntu-latest outputs: - changes: ${{ steps.changed_files.outputs.any_modified }} nf_test_files: ${{ steps.list.outputs.nf_test_files }} steps: - uses: actions/setup-python@v4 @@ -39,49 +38,14 @@ jobs: with: fetch-depth: 0 - - uses: tj-actions/changed-files@v42 - id: changed_files - with: - dir_names: "true" - output_renamed_files_as_deleted_and_added: "true" - # Define list of additional rules for testing paths - # Mostly, we define additional 'pipeline' or 'all' tests here - files_yaml: | - ".": - - nf-test.config - - nextflow.config - tests: - - assets/* - - bin/* - - conf/* - - main.nf - - nextflow_schema.json - - files_ignore: | - .git* - .gitpod.yml - .prettierignore - .prettierrc.yml - **.md - **.png - modules.json - pyproject.toml - tower.yml - - - name: debug - run: | - echo ${{ steps.changed_files.outputs.any_modified }} - echo ${{ steps.changed_files.outputs.all_changed_files }} - echo ${{ steps.changed_files.outputs.changed_keys }} - - name: nf-test list nf_test_files id: list - if: ${{ steps.changed_files.outputs.any_modified }} run: | echo nf_test_files=$(python \ .github/python/find_changed_files.py \ -t pipeline workflow process \ - -p ${{ steps.changed_files.outputs.all_changed_files }} ${{ steps.changed_files.outputs.changed_keys }} \ + --head_ref ${{ github.head_ref }} \ + --base_ref ${{ github.base_ref }} \ ) >> $GITHUB_OUTPUT - name: debug2 From 8e399c03715438da03b8ad07aea5f6def28444fc Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 28 Feb 2024 00:03:48 +0000 Subject: [PATCH 11/16] Install gitpython in CI --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fabb61a..b49d067 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,11 @@ jobs: with: fetch-depth: 0 + - name: Install gitpython find changed files + run: | + python -m pip install --upgrade pip + pip install gitpython + - name: nf-test list nf_test_files id: list run: | From 152bb4ba87caea6356e985a96673f819b8c818bf Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 28 Feb 2024 00:13:00 +0000 Subject: [PATCH 12/16] Changed branch1 to use head ref --- .github/python/find_changed_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index a96fc7c..908b7be 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -83,7 +83,7 @@ def find_files(branch1: str, branch2: str, ignore: list[str]) -> list[Path]: # create repo repo = Repo(".") # identify commit on branch1 - branch1_commit = repo.commit(branch1) + branch1_commit = repo.commit(repo.head.ref) # identify commit on branch2 branch2_commit = repo.commit(branch2) # compare two branches From 71e4d40f0c5b2c0bc5472fd5fa57e3985802b67a Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 28 Feb 2024 05:04:05 +0000 Subject: [PATCH 13/16] Try different github variable --- .github/python/find_changed_files.py | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index 908b7be..a96fc7c 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -83,7 +83,7 @@ def find_files(branch1: str, branch2: str, ignore: list[str]) -> list[Path]: # create repo repo = Repo(".") # identify commit on branch1 - branch1_commit = repo.commit(repo.head.ref) + branch1_commit = repo.commit(branch1) # identify commit on branch2 branch2_commit = repo.commit(branch2) # compare two branches diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b49d067..2e7eec8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: echo nf_test_files=$(python \ .github/python/find_changed_files.py \ -t pipeline workflow process \ - --head_ref ${{ github.head_ref }} \ + --head_ref ${{ github.ref }} \ --base_ref ${{ github.base_ref }} \ ) >> $GITHUB_OUTPUT From b79274ed755aeaa1a865d9ca88325b590ee99fd4 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 28 Feb 2024 05:15:55 +0000 Subject: [PATCH 14/16] Use commit sha --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e7eec8..dc01603 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: echo nf_test_files=$(python \ .github/python/find_changed_files.py \ -t pipeline workflow process \ - --head_ref ${{ github.ref }} \ + --head_ref ${{ github.sha }} \ --base_ref ${{ github.base_ref }} \ ) >> $GITHUB_OUTPUT From 3fff7f0d14db11a583bcace2fa976f4659270823 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 28 Feb 2024 05:17:27 +0000 Subject: [PATCH 15/16] Add origin prefix to base_ref --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc01603..b09601a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: .github/python/find_changed_files.py \ -t pipeline workflow process \ --head_ref ${{ github.sha }} \ - --base_ref ${{ github.base_ref }} \ + --base_ref origin/${{ github.base_ref }} \ ) >> $GITHUB_OUTPUT - name: debug2 From 36bd5615fe91d9e42aaa478c0cb214673934baa9 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 28 Feb 2024 05:26:40 +0000 Subject: [PATCH 16/16] Run CI if python list isn't empty --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b09601a..663b87f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,14 +53,14 @@ jobs: --base_ref origin/${{ github.base_ref }} \ ) >> $GITHUB_OUTPUT - - name: debug2 + - name: debug run: | echo ${{ steps.list.outputs.nf_test_files }} test: name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} needs: [changes] - if: needs.changes.outputs.changes + if: needs.changes.outputs.nf_test_files != '[]' runs-on: ubuntu-latest strategy: fail-fast: false