From 3916aaa810adc9ce672a82cff0fccc6fdb468738 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:12:36 +0200 Subject: [PATCH 01/13] Add an nf-test CI --- .github/workflows/ci.yml | 133 ++++++++++++++++++++++++++++++++++++--- CHANGELOG.md | 1 + 2 files changed, 127 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 361fca04fe..a67388d8f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,34 +11,40 @@ on: - master - dev +env: + NXF_ANSI_LOG: false + NFTEST_VER: "0.8.1" + # Cancel if a newer run is started concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - changes: - name: Check for changes + pytest-changes: + name: Check for changes (pytest) runs-on: ubuntu-latest outputs: # Expose matched filters as job 'tags' output variable tags: ${{ steps.filter.outputs.changes }} steps: - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 + + - uses: frouioui/paths-filter@main id: filter with: - filters: "tests/config/tags.yml" + filters: ".github/tags.yml" + token: "" - test: + pytest: name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} runs-on: ubuntu-latest needs: changes - if: needs.changes.outputs.tags != '[]' + if: needs.pytest-changes.outputs.tags != '[]' strategy: fail-fast: false matrix: - tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] + tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] profile: ["docker"] # profile: ["docker", "singularity", "conda"] TEST_DATA_BASE: @@ -50,6 +56,7 @@ jobs: NXF_ANSI_LOG: false TEST_DATA_BASE: "${{ github.workspace }}/test-datasets" SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + steps: - name: Check out pipeline code uses: actions/checkout@v3 @@ -154,6 +161,118 @@ jobs: !/home/runner/pytest_workflow_*/*/work/conda !/home/runner/pytest_workflow_*/*/work/singularity + nftest-changes: + name: Check for changes (nf-test) + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + steps: + - name: Combine all tags.yml files + id: get_username + run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml + + - name: debug + run: cat .github/tags.yml + + - uses: frouioui/paths-filter@main + id: filter + with: + filters: ".github/tags.yml" + token: "" + + nftest: + name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} + runs-on: ubuntu-latest + needs: changes + if: needs.nftest-changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.nftest-changes.outputs.tags) }}"] + profile: ["docker"] + # profile: ["docker", "singularity", "conda"] + TEST_DATA_BASE: + - "test-datasets/data" + NXF_VER: + - "23.04.0" + - "latest-everything" + env: + NXF_ANSI_LOG: false + TEST_DATA_BASE: "${{ github.workspace }}/test-datasets" + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v3 + with: + distribution: "temurin" + java-version: "17" + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v1 + + - name: Cache nf-test installation + id: cache-software + uses: actions/cache@v3 + with: + path: | + /usr/local/bin/nf-test + /home/runner/.nf-test/nf-test.jar + key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest + + - name: Install nf-test + if: steps.cache-software.outputs.cache-hit != 'true' + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ + + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda setup + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_LICENSE_BASE64 != null + run: | + nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }} + nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }} + SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d) + SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d) + SENTIEON_AUTH_DATA=$(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0) + nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64 + + # Test the module + - name: Run nf-test + run: | + nf-test test \ + --profile=${{ matrix.profile }} \ + --tag ${{ matrix.tags }} \ + --tap=test.tap \ + --verbose + confirm-pass: runs-on: ubuntu-latest needs: diff --git a/CHANGELOG.md b/CHANGELOG.md index 86db9aae32..77ac51444e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Rapaselet is a delta formed by the Rapaätno river between the Bielloriehppe mas - [#1173](https://github.com/nf-core/sarek/pull/1173) - CI tests for VQSR track with stub runs - [#1122](https://github.com/nf-core/sarek/pull/1122), [#1196](https://github.com/nf-core/sarek/pull/1196) - Add `annotation cache` functionality - [#1184](https://github.com/nf-core/sarek/pull/1184) - Stub-based CI-test of Sentieon joint-germline variant-calling with VQSR +- NF-TEST CI ### Changed From 703482d846824e1bbf1ea4a1bf9e21aace043986 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:16:17 +0200 Subject: [PATCH 02/13] fix --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a67388d8f2..ca6d1a3b1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: pytest: name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} runs-on: ubuntu-latest - needs: changes + needs: pytest-changes if: needs.pytest-changes.outputs.tags != '[]' strategy: fail-fast: false @@ -183,7 +183,7 @@ jobs: nftest: name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} runs-on: ubuntu-latest - needs: changes + needs: nftest-changes if: needs.nftest-changes.outputs.tags != '[]' strategy: fail-fast: false From 5fb95e74feaf67cbd1978d2d62fe742458612fc1 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:17:24 +0200 Subject: [PATCH 03/13] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77ac51444e..60d77c29df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ Rapaselet is a delta formed by the Rapaätno river between the Bielloriehppe mas - [#1173](https://github.com/nf-core/sarek/pull/1173) - CI tests for VQSR track with stub runs - [#1122](https://github.com/nf-core/sarek/pull/1122), [#1196](https://github.com/nf-core/sarek/pull/1196) - Add `annotation cache` functionality - [#1184](https://github.com/nf-core/sarek/pull/1184) - Stub-based CI-test of Sentieon joint-germline variant-calling with VQSR -- NF-TEST CI +- [#1288](https://github.com/nf-core/sarek/pull/1288) - Add nf-test continuous integration. ### Changed From 4680ff70c22ea80537b667482d06008bc1d8b630 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:18:16 +0200 Subject: [PATCH 04/13] fixxx --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca6d1a3b1c..4ed26dee82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -276,7 +276,8 @@ jobs: confirm-pass: runs-on: ubuntu-latest needs: - - test + - pytest + - nftest if: always() steps: - name: All tests ok From 514e18c67ff283c8c2ca826ff59cc0221b6fcdfa Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 09:21:55 +0200 Subject: [PATCH 05/13] Fix pytest tags --- .github/workflows/ci.yml | 2 +- nextflow.config | 18 ++++++++++++++++++ tests/tags.yml | 5 +++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tests/tags.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ed26dee82..8d584ee553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - uses: frouioui/paths-filter@main id: filter with: - filters: ".github/tags.yml" + filters: "tests/config/tags.yml" token: "" pytest: diff --git a/nextflow.config b/nextflow.config index e9ec2a66da..1cb0d5b768 100644 --- a/nextflow.config +++ b/nextflow.config @@ -134,6 +134,24 @@ params { validate_params = true } +tower { + enabled = true + workspaceId = '70508760960023' +} + +wave { + enabled = true +} + +fusion { + enabled = true +} + +wave.build.repository = 'seqeralabsdev.azurecr.io/wave' +wave.build.cacheRepository = 'seqeralabsdev.azurecr.io/cache' +wave.strategy = 'conda,dockerfile,spack,container' +wave.freeze = true + // Load base.config by default for all pipelines includeConfig 'conf/base.config' diff --git a/tests/tags.yml b/tests/tags.yml new file mode 100644 index 0000000000..0b52c379ca --- /dev/null +++ b/tests/tags.yml @@ -0,0 +1,5 @@ +pipeline_sarek: + - "**.nf" + - "**.config" + - "**.nf.test" + - "**.json" From c180a91eadebc334576fb5bb0921d8fd69b60328 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 09:41:12 +0200 Subject: [PATCH 06/13] Revert config item --- nextflow.config | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/nextflow.config b/nextflow.config index 1cb0d5b768..e9ec2a66da 100644 --- a/nextflow.config +++ b/nextflow.config @@ -134,24 +134,6 @@ params { validate_params = true } -tower { - enabled = true - workspaceId = '70508760960023' -} - -wave { - enabled = true -} - -fusion { - enabled = true -} - -wave.build.repository = 'seqeralabsdev.azurecr.io/wave' -wave.build.cacheRepository = 'seqeralabsdev.azurecr.io/cache' -wave.strategy = 'conda,dockerfile,spack,container' -wave.freeze = true - // Load base.config by default for all pipelines includeConfig 'conf/base.config' From 52e70dc10810ce2e8c45200b5ae3331a860d0715 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 10:48:50 +0200 Subject: [PATCH 07/13] Add empty tags file --- .github/tags.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/tags.yml diff --git a/.github/tags.yml b/.github/tags.yml new file mode 100644 index 0000000000..e69de29bb2 From 22bddc963c4676cd1573c5d2f3f379b028e7401c Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:08:37 +0200 Subject: [PATCH 08/13] quote run --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d584ee553..a7ffb35e2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,7 +169,7 @@ jobs: steps: - name: Combine all tags.yml files id: get_username - run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml + run: 'find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml' - name: debug run: cat .github/tags.yml From 06d9dda8c017ccdf6894f10c75903492ba361fa5 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:14:10 +0200 Subject: [PATCH 09/13] checkout repo before running --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7ffb35e2f..6139e9a113 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,10 +166,13 @@ jobs: runs-on: ubuntu-latest outputs: tags: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v3 + - name: Combine all tags.yml files - id: get_username - run: 'find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml' + id: get_tags + run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml - name: debug run: cat .github/tags.yml From 6848818a5d23d5e0c2795f49721a06dec55e2ae2 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:33:39 +0200 Subject: [PATCH 10/13] Add nf-test --- .gitignore | 12 +++++++----- nf-test.config | 8 ++++++++ tests/main.nf.test | 28 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 nf-test.config create mode 100644 tests/main.nf.test diff --git a/.gitignore b/.gitignore index e90c12ab54..936a50d19f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ *.code-workspace +*.pyc .nextflow* -work/ -data/ -results/ .DS_Store -testing/ +.nf-test.log +.nf-test/ +data/ testing* -*.pyc +testing/ test-datasets/ +results/ +work/ \ No newline at end of file diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 0000000000..c1ffc5e16c --- /dev/null +++ b/nf-test.config @@ -0,0 +1,8 @@ +config { + + testsDir "tests" + workDir ".nf-test" + configFile "conf/test.config" + profile "test" + +} diff --git a/tests/main.nf.test b/tests/main.nf.test new file mode 100644 index 0000000000..33d3577928 --- /dev/null +++ b/tests/main.nf.test @@ -0,0 +1,28 @@ +nextflow_pipeline { + + name "Test pipeline" + script "../main.nf" + tag "pipeline" + tag "pipeline_sarek" + + test("Run with profile test") { + + when { + params { + outdir = "results" + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + input = "$projectDir/tests/csv/3.0/fastq_pair.csv" + validationSchemaIgnoreParams = 'test_data_base,test_data' + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + +} \ No newline at end of file From 2873e2baeedcdcc6b1b24a6445f2b906869df50e Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:50:17 +0200 Subject: [PATCH 11/13] Add .gitignore to files_unchanged --- .nf-core.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.nf-core.yml b/.nf-core.yml index a6a655b221..b67cd20f88 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -4,6 +4,7 @@ lint: files_exist: - conf/modules.config files_unchanged: + - .gitignore - assets/nf-core-sarek_logo_light.png - docs/images/nf-core-sarek_logo_dark.png - docs/images/nf-core-sarek_logo_light.png From de668ba066044c99738814f5a3ed2bc7e5751b9d Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:51:35 +0200 Subject: [PATCH 12/13] Remove extra tags.yml --- .github/tags.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .github/tags.yml diff --git a/.github/tags.yml b/.github/tags.yml deleted file mode 100644 index e69de29bb2..0000000000 From 9f7d6855aa2a39fd2a7a218e4aca3ee53e573888 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:54:46 +0200 Subject: [PATCH 13/13] eclint --- tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/main.nf.test b/tests/main.nf.test index 33d3577928..1b4e18a2e0 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -25,4 +25,4 @@ nextflow_pipeline { } } -} \ No newline at end of file +}