From 02a96cab1c40560fc31698403899a472462e869d Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sun, 14 Aug 2022 08:09:34 +0200 Subject: [PATCH 001/854] Add new custom groovy library to generate a methods section for MultiQC --- .../assets/methods_description_template.yml | 25 +++++++++++++++++++ .../assets/multiqc_config.yml | 6 +++-- .../lib/WorkflowPipeline.groovy | 15 +++++++++++ nf_core/pipeline-template/nextflow.config | 8 +++--- .../pipeline-template/workflows/pipeline.nf | 9 +++++-- 5 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 nf_core/pipeline-template/assets/methods_description_template.yml diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml new file mode 100644 index 0000000000..e330e51615 --- /dev/null +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -0,0 +1,25 @@ +id: "${workflow.manifest.name.replace('/', '-')}-methods-description" +description: "Suggested methods description text of pipeline usage" +section_name: "${ workflow.manifest.name } Methods Description" +section_href: "https://github.com/${workflow.manifest.name}" +plot_type: "html" +## TODO nf-core: Update the HTML below to your prefered methods description, e.g. add publication citation for this pipeline +## You inject any metadata in the Nextflow '${workflow}' object +data: | + Methods +

Data was processed using the ${workflow.manifest.name} (v${workflow.manifest.version}, doi: ${workflow.manifest.doi}) of the nf-core collection of workflows (Ewels et al. 2020).

+

The pipeline was executed with Nextflow (v${workflow.nextflow.version}; Di Tommaso et al. 2017) with the following command:

+ 
${workflow.commandLine}
+ References + +
+ Notes:
+ +
diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index a9cc6cdb35..89c8b9146d 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -3,9 +3,11 @@ report_comment: > analysis pipeline.{% if branded %} For information about how to interpret these results, please see the documentation.{% endif %} report_section_order: - software_versions: + "{{ name.lower().replace('/', '-') }}-methods-description": order: -1000 - "{{ name.lower().replace('/', '-') }}-summary": + software_versions: order: -1001 + "{{ name.lower().replace('/', '-') }}-summary": + order: -1002 export_plots: true diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index ba9199e6fc..618af77050 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the workflow/{{ short_name }}.nf in the {{ name }} pipeline // +import groovy.text.SimpleTemplateEngine + class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // @@ -45,6 +47,19 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { return yaml_file_text } + public static String methodsDescriptionText(run_workflow, mqc_methods_yaml) { + // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file + def meta = [:] + meta.workflow = run_workflow.toMap() + + def methods_text = mqc_methods_yaml.text + + def engine = new SimpleTemplateEngine() + def description_html = engine.createTemplate(methods_text).make(meta) + + return description_html + } + {%- if igenomes -%} // // Exit pipeline if incorrect --genome key provided diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index fb9db8f03d..336cc86355 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -21,9 +21,10 @@ params { {% endif -%} // MultiQC options - multiqc_config = null - multiqc_title = null - max_multiqc_email_size = '25.MB' + multiqc_config = null + multiqc_title = null + max_multiqc_email_size = '25.MB' + multiqc_methods_description = null // Boilerplate options outdir = null @@ -191,6 +192,7 @@ manifest { mainScript = 'main.nf' nextflowVersion = '!>=21.10.3' version = '{{ version }}' + doi = '' } // Load modules.config for DSL2 module specific options diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 97e80b7c3a..57a56f9734 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -23,8 +23,9 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() +ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) +ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() +ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -89,10 +90,14 @@ workflow {{ short_name|upper }} { workflow_summary = Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.paramsSummaryMultiqc(workflow, summary_params) ch_workflow_summary = Channel.value(workflow_summary) + methods_description = WorkflowTestpipeline.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) + ch_methods_description = Channel.value(methods_description) + ch_multiqc_files = Channel.empty() ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) From 63e3df6f00237e6956fe0e9661dd97d50c4e894e Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sun, 14 Aug 2022 08:17:54 +0200 Subject: [PATCH 002/854] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd26d065b..728396bb03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Make `nf-core create` fail if Git default branch name is dev or TEMPLATE ([#1705](https://github.com/nf-core/tools/pull/1705)) - Convert `console` snippets to `bash` snippets in the template where applicable ([#1729](https://github.com/nf-core/tools/pull/1729)) - Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728)) +- Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ### Linting From d9a30d54ea84749dd950a19356d4109a0efa0ac5 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sun, 14 Aug 2022 08:22:36 +0200 Subject: [PATCH 003/854] Add to schema --- nf_core/pipeline-template/nextflow_schema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 5cd8ac489a..0f261ea55f 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -219,6 +219,12 @@ "fa_icon": "fas fa-cog", "hidden": true }, + "multiqc_methods_description": { + "type": "string", + "description": "Custom MultiQC yaml file containing HTML including a methods description.", + "fa_icon": "fas fa-cog", + "hidden": true + }, "tracedir": { "type": "string", "description": "Directory to keep pipeline Nextflow logs and reports.", From de682fb329bc7085885259e2660eff200f02cc8e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 17 Aug 2022 11:18:02 +0200 Subject: [PATCH 004/854] Update nf_core/pipeline-template/assets/methods_description_template.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .../pipeline-template/assets/methods_description_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml index e330e51615..e0d6f5ba26 100644 --- a/nf_core/pipeline-template/assets/methods_description_template.yml +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -1,5 +1,5 @@ id: "${workflow.manifest.name.replace('/', '-')}-methods-description" -description: "Suggested methods description text of pipeline usage" +description: "Suggested methods description text of pipeline usage." section_name: "${ workflow.manifest.name } Methods Description" section_href: "https://github.com/${workflow.manifest.name}" plot_type: "html" From b2c5f1e8ba71905a6d83e3192aaa30e43317b4b3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 17 Aug 2022 15:04:16 +0200 Subject: [PATCH 005/854] Apply suggestions from code review Co-authored-by: Phil Ewels --- CHANGELOG.md | 2 +- .../assets/methods_description_template.yml | 30 +++++++++---------- .../assets/multiqc_config.yml | 4 +-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 728396bb03..3683071e56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Make `nf-core create` fail if Git default branch name is dev or TEMPLATE ([#1705](https://github.com/nf-core/tools/pull/1705)) - Convert `console` snippets to `bash` snippets in the template where applicable ([#1729](https://github.com/nf-core/tools/pull/1729)) - Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728)) -- Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications +- Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) ### Linting diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml index e0d6f5ba26..ac15db072b 100644 --- a/nf_core/pipeline-template/assets/methods_description_template.yml +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -1,25 +1,25 @@ -id: "${workflow.manifest.name.replace('/', '-')}-methods-description" +id: "{{ name_noslash }}-methods-description" description: "Suggested methods description text of pipeline usage." -section_name: "${ workflow.manifest.name } Methods Description" -section_href: "https://github.com/${workflow.manifest.name}" +section_name: "{{ name }} Methods Description" +section_href: "https://github.com/{{ name }}" plot_type: "html" ## TODO nf-core: Update the HTML below to your prefered methods description, e.g. add publication citation for this pipeline ## You inject any metadata in the Nextflow '${workflow}' object data: | - Methods -

Data was processed using the ${workflow.manifest.name} (v${workflow.manifest.version}, doi: ${workflow.manifest.doi}) of the nf-core collection of workflows (Ewels et al. 2020).

-

The pipeline was executed with Nextflow (v${workflow.nextflow.version}; Di Tommaso et al. 2017) with the following command:

+

Methods

+

Data was processed using {{ name }} v${workflow.manifest.version} (DOI: ${workflow.manifest.doi}) of the nf-core collection of workflows (Ewels et al., 2020).

+

The pipeline was executed with Nextflow v${workflow.nextflow.version} (Di Tommaso et al., 2017) with the following command:

${workflow.commandLine}
- References +

References

    -
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. https://doi.org/10.1038/nbt.3820
    • -
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. https://doi.org/10.1038/s41587-020-0439-x
    • +
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. https://doi.org/10.1038/nbt.3820
    • +
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. https://doi.org/10.1038/s41587-020-0439-x
- Notes:
-
    -
  • If present, make sure to replace the pipeline DOI with correct reference information.
    • -
  • The command above does not include parameters contained in any configs pr profiles that may have been used. Ensure the config file is also uploaded with your publication!
    • -
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
    • -
+
Notes:
+
    +
  • If present, make sure to replace the pipeline DOI with correct reference information.
    • +
  • The command above does not include parameters contained in any configs pr profiles that may have been used. Ensure the config file is also uploaded with your publication!
    • +
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
    • +
diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index 89c8b9146d..440b0b9a3a 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -3,11 +3,11 @@ report_comment: > analysis pipeline.{% if branded %} For information about how to interpret these results, please see the documentation.{% endif %} report_section_order: - "{{ name.lower().replace('/', '-') }}-methods-description": + "{{ name_noslash }}-methods-description": order: -1000 software_versions: order: -1001 - "{{ name.lower().replace('/', '-') }}-summary": + "{{ name_noslash }}-summary": order: -1002 export_plots: true From 6f13ab2795182c9be89c1919a63d203f958fa200 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 23 Aug 2022 15:29:06 +0200 Subject: [PATCH 006/854] Apply suggestions from code review Co-authored-by: Phil Ewels --- .../assets/methods_description_template.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml index ac15db072b..7ee7c142a2 100644 --- a/nf_core/pipeline-template/assets/methods_description_template.yml +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -12,14 +12,14 @@ data: | 
${workflow.commandLine}

References

    -
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. https://doi.org/10.1038/nbt.3820
    • -
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. https://doi.org/10.1038/s41587-020-0439-x
    • +
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. https://doi.org/10.1038/nbt.3820
    • +
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. https://doi.org/10.1038/s41587-020-0439-x
-
+
Notes:
  • If present, make sure to replace the pipeline DOI with correct reference information.
  • The command above does not include parameters contained in any configs pr profiles that may have been used. Ensure the config file is also uploaded with your publication!
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
-
+ From 4da8e9f0969f7cec0ca84383b4827b5546bb8d0b Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 23 Aug 2022 15:41:02 +0200 Subject: [PATCH 007/854] Update nf_core/pipeline-template/assets/methods_description_template.yml --- .../pipeline-template/assets/methods_description_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml index 7ee7c142a2..d3a78d3fa3 100644 --- a/nf_core/pipeline-template/assets/methods_description_template.yml +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -19,7 +19,7 @@ data: |
Notes:
  • If present, make sure to replace the pipeline DOI with correct reference information.
    • -
  • The command above does not include parameters contained in any configs pr profiles that may have been used. Ensure the config file is also uploaded with your publication!
    • +
  • The command above does not include parameters contained in any configs or profiles that may have been used. Ensure the config file is also uploaded with your publication!
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
From 98eacbc0775e38ce3ed1cc68a60c717d2997a32f Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Tue, 23 Aug 2022 16:41:59 +0200 Subject: [PATCH 008/854] Add conditional DOI --- .../pipeline-template/assets/methods_description_template.yml | 4 ++-- nf_core/pipeline-template/lib/WorkflowPipeline.groovy | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml index 7ee7c142a2..daa6fdf47c 100644 --- a/nf_core/pipeline-template/assets/methods_description_template.yml +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -7,7 +7,7 @@ plot_type: "html" ## You inject any metadata in the Nextflow '${workflow}' object data: |

Methods

-

Data was processed using {{ name }} v${workflow.manifest.version} (DOI: ${workflow.manifest.doi}) of the nf-core collection of workflows (Ewels et al., 2020).

+

Data was processed using {{ name }} v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (Ewels et al., 2020).

The pipeline was executed with Nextflow v${workflow.nextflow.version} (Di Tommaso et al., 2017) with the following command:

${workflow.commandLine}

References

@@ -19,7 +19,7 @@ data: |
Notes:
  • If present, make sure to replace the pipeline DOI with correct reference information.
    • -
  • The command above does not include parameters contained in any configs pr profiles that may have been used. Ensure the config file is also uploaded with your publication!
    • +
  • The command above does not include parameters contained in any configs or profiles that may have been used. Ensure the config file is also uploaded with your publication!
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 618af77050..2cd2dceff4 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -51,6 +51,9 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file def meta = [:] meta.workflow = run_workflow.toMap() + meta["manifest_map"] = run_workflow.manifest.toMap() + + meta["doi_text"] = meta.manifest_map.doi ? "(DOI: ${meta.manifest_map.doi})" : "" def methods_text = mqc_methods_yaml.text From d94c50b7ae886ac741702d45412a877ff8156922 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 25 Aug 2022 09:04:04 +0000 Subject: [PATCH 009/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-modules-lint.svg | 252 ++++++++------------ docs/images/nf-core-modules-list-local.svg | 132 +++++----- docs/images/nf-core-modules-list-remote.svg | 132 +++++----- docs/images/nf-core-modules-remove.svg | 72 +++--- docs/images/nf-core-schema-build.svg | 84 +++---- docs/images/nf-core-schema-lint.svg | 78 +++--- 6 files changed, 347 insertions(+), 403 deletions(-) diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index d0503aac4d..8a4d1f6113 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:195 -INFO     Linting module: 'multiqc'__init__.py:199 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ -│                                           ╷                ╷                 │ -│Module name                              │File path     │Test message   │ -│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ -│multiqc│modules/multi…│Conda update: │ -│││bioconda::mult…│ -│││1.10 -> 1.13a│ -│                                           ╵                ╵                 │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:195 +INFO     Linting module: 'multiqc'__init__.py:199 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ +│                                           ╷                ╷                 │ +│Module name                              │File path     │Test message   │ +│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ +│multiqc│modules/multi…│Conda update: │ +│││bioconda::mult…│ +│││1.10 -> 1.13a│ +│                                           ╵                ╵                 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index e53839201b..9297ec424b 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2388969700-matrix { + .terminal-1989200096-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2388969700-title { + .terminal-1989200096-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2388969700-r1 { fill: #c5c8c6 } -.terminal-2388969700-r2 { fill: #98a84b } -.terminal-2388969700-r3 { fill: #9a9b99 } -.terminal-2388969700-r4 { fill: #608ab1 } -.terminal-2388969700-r5 { fill: #d0b344 } -.terminal-2388969700-r6 { fill: #868887 } -.terminal-2388969700-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2388969700-r8 { fill: #868887;font-style: italic; } + .terminal-1989200096-r1 { fill: #c5c8c6 } +.terminal-1989200096-r2 { fill: #98a84b } +.terminal-1989200096-r3 { fill: #9a9b99 } +.terminal-1989200096-r4 { fill: #608ab1 } +.terminal-1989200096-r5 { fill: #d0b344 } +.terminal-1989200096-r6 { fill: #868887 } +.terminal-1989200096-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1989200096-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules installed in '.':                                   list.py:128 - -┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ -┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules installed in '.':                                   list.py:124 + +┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 328d5f100f..e26dcf52b8 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-1590927372-matrix { + .terminal-1871028241-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1590927372-title { + .terminal-1871028241-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1590927372-r1 { fill: #c5c8c6 } -.terminal-1590927372-r2 { fill: #98a84b } -.terminal-1590927372-r3 { fill: #9a9b99 } -.terminal-1590927372-r4 { fill: #608ab1 } -.terminal-1590927372-r5 { fill: #d0b344 } -.terminal-1590927372-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1590927372-r7 { fill: #868887 } -.terminal-1590927372-r8 { fill: #868887;font-style: italic; } + .terminal-1871028241-r1 { fill: #c5c8c6 } +.terminal-1871028241-r2 { fill: #98a84b } +.terminal-1871028241-r3 { fill: #9a9b99 } +.terminal-1871028241-r4 { fill: #608ab1 } +.terminal-1871028241-r5 { fill: #d0b344 } +.terminal-1871028241-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1871028241-r7 { fill: #868887 } +.terminal-1871028241-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):            list.py:123 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):            list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 2c9d755ab1..a8b3b7a17a 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-623533593-matrix { + .terminal-623664666-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-623533593-title { + .terminal-623664666-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-623533593-r1 { fill: #c5c8c6 } -.terminal-623533593-r2 { fill: #98a84b } -.terminal-623533593-r3 { fill: #9a9b99 } -.terminal-623533593-r4 { fill: #608ab1 } -.terminal-623533593-r5 { fill: #d0b344 } -.terminal-623533593-r6 { fill: #868887 } + .terminal-623664666-r1 { fill: #c5c8c6 } +.terminal-623664666-r2 { fill: #98a84b } +.terminal-623664666-r3 { fill: #9a9b99 } +.terminal-623664666-r4 { fill: #608ab1 } +.terminal-623664666-r5 { fill: #d0b344 } +.terminal-623664666-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Removing abacas                                            remove.py:51 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Removing abacas                                            remove.py:52 diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 3d1e555dc6..67ba38633a 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-4157447415-matrix { + .terminal-4168195321-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4157447415-title { + .terminal-4168195321-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4157447415-r1 { fill: #c5c8c6 } -.terminal-4157447415-r2 { fill: #98a84b } -.terminal-4157447415-r3 { fill: #9a9b99 } -.terminal-4157447415-r4 { fill: #608ab1 } -.terminal-4157447415-r5 { fill: #d0b344 } -.terminal-4157447415-r6 { fill: #98a84b;font-weight: bold } -.terminal-4157447415-r7 { fill: #868887 } -.terminal-4157447415-r8 { fill: #868887;font-weight: bold } -.terminal-4157447415-r9 { fill: #4e707b;font-weight: bold } -.terminal-4157447415-r10 { fill: #68a0b3;font-weight: bold } + .terminal-4168195321-r1 { fill: #c5c8c6 } +.terminal-4168195321-r2 { fill: #98a84b } +.terminal-4168195321-r3 { fill: #9a9b99 } +.terminal-4168195321-r4 { fill: #608ab1 } +.terminal-4168195321-r5 { fill: #d0b344 } +.terminal-4168195321-r6 { fill: #98a84b;font-weight: bold } +.terminal-4168195321-r7 { fill: #868887 } +.terminal-4168195321-r8 { fill: #868887;font-weight: bold } +.terminal-4168195321-r9 { fill: #4e707b;font-weight: bold } +.terminal-4168195321-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 27 params)schema.py:95 -INFO     Writing schema with 28 params: './nextflow_schema.json'schema.py:173 + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 +INFO     Writing schema with 29 params: './nextflow_schema.json'schema.py:173 diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index 0e70f4518b..fcb19b9919 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-691504655-matrix { + .terminal-693601808-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-691504655-title { + .terminal-693601808-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-691504655-r1 { fill: #c5c8c6 } -.terminal-691504655-r2 { fill: #98a84b } -.terminal-691504655-r3 { fill: #9a9b99 } -.terminal-691504655-r4 { fill: #608ab1 } -.terminal-691504655-r5 { fill: #d0b344 } -.terminal-691504655-r6 { fill: #98a84b;font-weight: bold } -.terminal-691504655-r7 { fill: #868887 } -.terminal-691504655-r8 { fill: #868887;font-weight: bold } -.terminal-691504655-r9 { fill: #4e707b;font-weight: bold } + .terminal-693601808-r1 { fill: #c5c8c6 } +.terminal-693601808-r2 { fill: #98a84b } +.terminal-693601808-r3 { fill: #9a9b99 } +.terminal-693601808-r4 { fill: #608ab1 } +.terminal-693601808-r5 { fill: #d0b344 } +.terminal-693601808-r6 { fill: #98a84b;font-weight: bold } +.terminal-693601808-r7 { fill: #868887 } +.terminal-693601808-r8 { fill: #868887;font-weight: bold } +.terminal-693601808-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 29 params)schema.py:95 From 708a9b0d8666ef6ee53da0ce1b482e8c12efeaf6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 25 Aug 2022 13:36:31 +0200 Subject: [PATCH 010/854] add option to remove git hosting from pipeline template --- nf_core/create.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 8e58306cd8..47d95f016c 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -58,6 +58,10 @@ def __init__( ) skippable_paths = { + "github": [ + ".github/", + ".gitignore", + ], "ci": [".github/workflows/"], "igenomes": ["conf/igenomes.config"], "branded": [ @@ -74,7 +78,7 @@ def __init__( self.name = self.template_params["name"] # Set fields used by the class methods - self.no_git = no_git + self.no_git = no_git if self.template_params["github"] else True self.force = force if outdir is None: outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) @@ -110,6 +114,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa # Define the different template areas, and what actions to take for each # if they are skipped template_areas = { + "github": {"name": "GitHub hosting", "file": True, "content": False}, "ci": {"name": "GitHub CI", "file": True, "content": False}, "github_badges": {"name": "GitHub badges", "file": False, "content": True}, "igenomes": {"name": "iGenomes config", "file": True, "content": True}, @@ -313,9 +318,10 @@ def render_template(self): # Make a logo and save it, if it is a nf-core pipeline self.make_pipeline_logo() else: - # Remove field mentioning nf-core docs - # in the github bug report template - self.remove_nf_core_in_bug_report_template() + if self.template_params["github"]: + # Remove field mentioning nf-core docs + # in the github bug report template + self.remove_nf_core_in_bug_report_template() # Update the .nf-core.yml with linting configurations self.fix_linting() From a6d223e9bd6a23ea81b7ae6794330e2d2660375a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 25 Aug 2022 13:58:17 +0200 Subject: [PATCH 011/854] add file bug_report.yml to files_exist when github hosting is not added to the pipeline --- nf_core/create.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 47d95f016c..619a7b6cc8 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -78,7 +78,9 @@ def __init__( self.name = self.template_params["name"] # Set fields used by the class methods - self.no_git = no_git if self.template_params["github"] else True + self.no_git = ( + no_git if self.template_params["github"] else True + ) # Set to True if template was configured without github hosting self.force = force if outdir is None: outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) @@ -401,6 +403,15 @@ def fix_linting(self): "multiqc_config": ["report_comment"], } + # Add GitHub hosting specific configurations + if not self.template_params["github"]: + lint_config["files_exist"].extend( + [ + ".github/ISSUE_TEMPLATE/bug_report.yml", + ] + ) + lint_config["files_unchanged"] = [".github/ISSUE_TEMPLATE/bug_report.yml"] + # Add CI specific configurations if not self.template_params["ci"]: lint_config["files_exist"].extend( From 8b7e1b191d98bd677e3e91ad97a0933801c3d853 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 25 Aug 2022 14:42:22 +0200 Subject: [PATCH 012/854] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f56a185ba7..50cb2c4609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Make `nf-core create` fail if Git default branch name is dev or TEMPLATE ([#1705](https://github.com/nf-core/tools/pull/1705)) - Convert `console` snippets to `bash` snippets in the template where applicable ([#1729](https://github.com/nf-core/tools/pull/1729)) - Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728)) +- Add customisation option to remove all GitHub support with `nf-core create` ([#1766](https://github.com/nf-core/tools/pull/1766)) ### Linting From cff17170336e8a02d63d6de847ff8db3406d807c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 25 Aug 2022 13:07:37 +0000 Subject: [PATCH 013/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-create.svg | 168 +++++++------ docs/images/nf-core-list-stars.svg | 104 ++++---- docs/images/nf-core-list.svg | 108 ++++----- docs/images/nf-core-modules-lint.svg | 252 ++++++++------------ docs/images/nf-core-modules-list-local.svg | 132 +++++----- docs/images/nf-core-modules-list-remote.svg | 132 +++++----- docs/images/nf-core-modules-remove.svg | 72 +++--- docs/images/nf-core-sync.svg | 98 ++++---- 8 files changed, 503 insertions(+), 563 deletions(-) diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index d90cdfdaa7..06a7be2c2c 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - - - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  -big omics technique" -a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:226 -INFO     Initialising pipeline git repository                      create.py:518 -INFO     Done. Remember to add a remote and push to GitHub:        create.py:525 - cd  -/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core --nextbigthing - git remote add origin  -git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                    -INFO     This will also push your newly created dev branch and the create.py:531 -         TEMPLATE branch for syncing.                               -INFO    !!!!!! IMPORTANT !!!!!!create.py:217 - -If you are interested in adding your pipeline to the  -nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  -WRITING ANY CODE! - -Please read:  -https://nf-co.re/developers/adding_pipelines#join-the-com -munity + + + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  +big omics technique" -a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:233 +INFO     Initialising pipeline git repository                      create.py:535 +INFO     Done. Remember to add a remote and push to GitHub:        create.py:542 + cd  +/home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin  +git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                    +INFO     This will also push your newly created dev branch and the create.py:548 +         TEMPLATE branch for syncing.                               +INFO    !!!!!! IMPORTANT !!!!!!create.py:224 + +If you are interested in adding your pipeline to the  +nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  +WRITING ANY CODE! + +Please read:  +https://nf-co.re/developers/adding_pipelines#join-the-com +munity diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index cf4b23118e..8586699882 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-2036675850-matrix { + .terminal-1937061120-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2036675850-title { + .terminal-1937061120-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2036675850-r1 { fill: #c5c8c6 } -.terminal-2036675850-r2 { fill: #98a84b } -.terminal-2036675850-r3 { fill: #9a9b99 } -.terminal-2036675850-r4 { fill: #608ab1 } -.terminal-2036675850-r5 { fill: #d0b344 } -.terminal-2036675850-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2036675850-r7 { fill: #868887 } -.terminal-2036675850-r8 { fill: #868887;font-style: italic; } + .terminal-1937061120-r1 { fill: #c5c8c6 } +.terminal-1937061120-r2 { fill: #98a84b } +.terminal-1937061120-r3 { fill: #9a9b99 } +.terminal-1937061120-r4 { fill: #608ab1 } +.terminal-1937061120-r5 { fill: #d0b344 } +.terminal-1937061120-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1937061120-r7 { fill: #868887 } +.terminal-1937061120-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ rnaseq      │   499 │       3.8.1 │    3 months │           - │ -            │ -│             │       │             │         ago │             │              │ -│ sarek       │   186 │       3.0.1 │  6 days ago │           - │ -            │ -│ chipseq     │   121 │       1.2.2 │ 1 years ago │           - │ -            │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ +┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ +│ rnaseq      │   499 │       3.8.1 │    3 months │           - │ -            │ +│             │       │             │         ago │             │              │ +│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ +│ chipseq     │   121 │       1.2.2 │ 1 years ago │           - │ -            │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 5b0721e32e..5752a8a5a6 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-2123714818-matrix { + .terminal-1065112585-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2123714818-title { + .terminal-1065112585-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2123714818-r1 { fill: #c5c8c6 } -.terminal-2123714818-r2 { fill: #98a84b } -.terminal-2123714818-r3 { fill: #9a9b99 } -.terminal-2123714818-r4 { fill: #608ab1 } -.terminal-2123714818-r5 { fill: #d0b344 } -.terminal-2123714818-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2123714818-r7 { fill: #868887 } -.terminal-2123714818-r8 { fill: #868887;font-style: italic; } + .terminal-1065112585-r1 { fill: #c5c8c6 } +.terminal-1065112585-r2 { fill: #98a84b } +.terminal-1065112585-r3 { fill: #9a9b99 } +.terminal-1065112585-r4 { fill: #608ab1 } +.terminal-1065112585-r5 { fill: #d0b344 } +.terminal-1065112585-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1065112585-r7 { fill: #868887 } +.terminal-1065112585-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ sarek       │   186 │       3.0.1 │  6 days ago │           - │ -            │ -│ epitopepre… │    22 │       2.1.0 │ 3 weeks ago │           - │ -            │ -│ eager       │    70 │       2.4.5 │ 3 weeks ago │           - │ -            │ -│ viralrecon  │    74 │         2.5 │    1 months │           - │ -            │ -│             │       │             │         ago │             │              │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ +┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ +│ mag         │    95 │       2.2.1 │ 2 hours ago │           - │ -            │ +│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ +│ epitopepre… │    22 │       2.1.0 │ 3 weeks ago │           - │ -            │ +│ eager       │    70 │       2.4.5 │ 3 weeks ago │           - │ -            │ +│ viralrecon  │    74 │         2.5 │    1 months │           - │ -            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index d0503aac4d..3d7917dbe1 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:195 -INFO     Linting module: 'multiqc'__init__.py:199 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ -│                                           ╷                ╷                 │ -│Module name                              │File path     │Test message   │ -│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ -│multiqc│modules/multi…│Conda update: │ -│││bioconda::mult…│ -│││1.10 -> 1.13a│ -│                                           ╵                ╵                 │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:195 +INFO     Linting module: 'multiqc'__init__.py:199 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ +│                                           ╷                ╷                 │ +│Module name                              │File path     │Test message   │ +│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ +│multiqc│modules/multi…│Conda update: │ +│││bioconda::mult…│ +│││1.10 -> 1.13a│ +│                                           ╵                ╵                 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index e53839201b..9297ec424b 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2388969700-matrix { + .terminal-1989200096-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2388969700-title { + .terminal-1989200096-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2388969700-r1 { fill: #c5c8c6 } -.terminal-2388969700-r2 { fill: #98a84b } -.terminal-2388969700-r3 { fill: #9a9b99 } -.terminal-2388969700-r4 { fill: #608ab1 } -.terminal-2388969700-r5 { fill: #d0b344 } -.terminal-2388969700-r6 { fill: #868887 } -.terminal-2388969700-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2388969700-r8 { fill: #868887;font-style: italic; } + .terminal-1989200096-r1 { fill: #c5c8c6 } +.terminal-1989200096-r2 { fill: #98a84b } +.terminal-1989200096-r3 { fill: #9a9b99 } +.terminal-1989200096-r4 { fill: #608ab1 } +.terminal-1989200096-r5 { fill: #d0b344 } +.terminal-1989200096-r6 { fill: #868887 } +.terminal-1989200096-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1989200096-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules installed in '.':                                   list.py:128 - -┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ -┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules installed in '.':                                   list.py:124 + +┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 328d5f100f..e26dcf52b8 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-1590927372-matrix { + .terminal-1871028241-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1590927372-title { + .terminal-1871028241-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1590927372-r1 { fill: #c5c8c6 } -.terminal-1590927372-r2 { fill: #98a84b } -.terminal-1590927372-r3 { fill: #9a9b99 } -.terminal-1590927372-r4 { fill: #608ab1 } -.terminal-1590927372-r5 { fill: #d0b344 } -.terminal-1590927372-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1590927372-r7 { fill: #868887 } -.terminal-1590927372-r8 { fill: #868887;font-style: italic; } + .terminal-1871028241-r1 { fill: #c5c8c6 } +.terminal-1871028241-r2 { fill: #98a84b } +.terminal-1871028241-r3 { fill: #9a9b99 } +.terminal-1871028241-r4 { fill: #608ab1 } +.terminal-1871028241-r5 { fill: #d0b344 } +.terminal-1871028241-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1871028241-r7 { fill: #868887 } +.terminal-1871028241-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):            list.py:123 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):            list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 2c9d755ab1..a8b3b7a17a 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-623533593-matrix { + .terminal-623664666-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-623533593-title { + .terminal-623664666-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-623533593-r1 { fill: #c5c8c6 } -.terminal-623533593-r2 { fill: #98a84b } -.terminal-623533593-r3 { fill: #9a9b99 } -.terminal-623533593-r4 { fill: #608ab1 } -.terminal-623533593-r5 { fill: #d0b344 } -.terminal-623533593-r6 { fill: #868887 } + .terminal-623664666-r1 { fill: #c5c8c6 } +.terminal-623664666-r2 { fill: #98a84b } +.terminal-623664666-r3 { fill: #9a9b99 } +.terminal-623664666-r4 { fill: #608ab1 } +.terminal-623664666-r5 { fill: #d0b344 } +.terminal-623664666-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Removing abacas                                            remove.py:51 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Removing abacas                                            remove.py:52 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 81f992bf71..57cfed4dd6 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Pipeline directory:                                          sync.py:95 -/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core-ne -xtbigthing -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 -INFO     Making a new template pipeline using pipeline variables     sync.py:223 + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Pipeline directory:                                          sync.py:95 +/home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 +INFO     Making a new template pipeline using pipeline variables     sync.py:223 From b32e2b05552a315e99cb10b55436c5e67366a034 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 25 Aug 2022 16:25:38 +0200 Subject: [PATCH 014/854] disable unnecessary pylint warnings --- nf_core/__main__.py | 4 ++-- nf_core/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3ef6888ef7..556515224b 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -415,7 +415,7 @@ def remote(ctx, keywords, json): default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def local(ctx, keywords, json, dir): +def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List modules installed locally in a pipeline """ @@ -666,7 +666,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") -def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version): +def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version): # pylint: disable=redefined-outer-name """ Lint one or more modules in a directory. diff --git a/nf_core/utils.py b/nf_core/utils.py index 1c65e45bb8..9321ff9629 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -409,7 +409,7 @@ class GitHub_API_Session(requests_cache.CachedSession): such as automatically setting up GitHub authentication if we can. """ - def __init__(self): + def __init__(self): # pylint: disable=super-init-not-called self.auth_mode = None self.return_ok = [200, 201] self.return_retry = [403] From a8a740e16be19acdaa4530ab1a4a9a1f62378821 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 25 Aug 2022 16:27:33 +0200 Subject: [PATCH 015/854] remove unnecessary f-strings --- nf_core/modules/update.py | 4 ++-- nf_core/schema.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 4644c2398e..124baccfb5 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -178,9 +178,9 @@ def update(self, module=None): if dry_run: if patch_relpath is not None: if patch_successful: - log.info(f"Current installation is compared against patched version in remote") + log.info("Current installation is compared against patched version in remote.") else: - log.warning(f"Current installation is compared against unpatched version in remote") + log.warning("Current installation is compared against unpatched version in remote.") # Compute the diffs for the module if self.save_diff_fn: log.info(f"Writing diff file for module '{module_fullname}' to '{self.save_diff_fn}'") diff --git a/nf_core/schema.py b/nf_core/schema.py index 1dab8c2b0e..6804183cb8 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -434,10 +434,10 @@ def check_for_input_mimetype(self): # Check that the input parameter is defined if "input" not in self.schema_params: - raise LookupError(f"Parameter `input` not found in schema") + raise LookupError("Parameter `input` not found in schema") # Check that the input parameter is defined in the right place if "input" not in self.schema.get("definitions", {}).get("input_output_options", {}).get("properties", {}): - raise LookupError(f"Parameter `input` is not defined in the correct subschema (input_output_options)") + raise LookupError("Parameter `input` is not defined in the correct subschema (input_output_options)") input_entry = self.schema["definitions"]["input_output_options"]["properties"]["input"] if "mimetype" not in input_entry: return None From 727a0b0b371588127195ff2dd68dfbe9288cd941 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 25 Aug 2022 16:27:44 +0200 Subject: [PATCH 016/854] use super init instead of repeated code --- nf_core/modules/lint/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 3910ef829e..096cac1f40 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -65,17 +65,12 @@ class ModuleLint(ModuleCommand): from .module_version import module_version def __init__(self, dir, fail_warned=False, remote_url=None, branch=None, no_pull=False): - self.dir = dir - try: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) - except LookupError as e: - raise UserWarning(e) + super().__init__(dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull) self.fail_warned = fail_warned self.passed = [] self.warned = [] self.failed = [] - self.modules_repo = ModulesRepo(remote_url, branch, no_pull) self.lint_tests = self.get_all_lint_tests(self.repo_type == "pipeline") if self.repo_type == "pipeline": From 798d415ce477645007f34ee58df82c533e3e0f71 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 25 Aug 2022 15:32:40 +0000 Subject: [PATCH 017/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-create.svg | 168 +++++++------ docs/images/nf-core-list-rna.svg | 118 ++++----- docs/images/nf-core-list-stars.svg | 104 ++++---- docs/images/nf-core-list.svg | 108 ++++----- docs/images/nf-core-modules-lint.svg | 252 ++++++++------------ docs/images/nf-core-modules-list-local.svg | 132 +++++----- docs/images/nf-core-modules-list-remote.svg | 132 +++++----- docs/images/nf-core-modules-remove.svg | 72 +++--- docs/images/nf-core-sync.svg | 98 ++++---- 9 files changed, 562 insertions(+), 622 deletions(-) diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index d90cdfdaa7..6b6667a5e9 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - - - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  -big omics technique" -a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:226 -INFO     Initialising pipeline git repository                      create.py:518 -INFO     Done. Remember to add a remote and push to GitHub:        create.py:525 - cd  -/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core --nextbigthing - git remote add origin  -git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                    -INFO     This will also push your newly created dev branch and the create.py:531 -         TEMPLATE branch for syncing.                               -INFO    !!!!!! IMPORTANT !!!!!!create.py:217 - -If you are interested in adding your pipeline to the  -nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  -WRITING ANY CODE! - -Please read:  -https://nf-co.re/developers/adding_pipelines#join-the-com -munity + + + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  +big omics technique" -a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:226 +INFO     Initialising pipeline git repository                      create.py:518 +INFO     Done. Remember to add a remote and push to GitHub:        create.py:525 + cd  +/home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin  +git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                    +INFO     This will also push your newly created dev branch and the create.py:531 +         TEMPLATE branch for syncing.                               +INFO    !!!!!! IMPORTANT !!!!!!create.py:217 + +If you are interested in adding your pipeline to the  +nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  +WRITING ANY CODE! + +Please read:  +https://nf-co.re/developers/adding_pipelines#join-the-com +munity diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 235b4a479a..e893848c09 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-4179116690-matrix { + .terminal-4114956945-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4179116690-title { + .terminal-4114956945-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4179116690-r1 { fill: #c5c8c6 } -.terminal-4179116690-r2 { fill: #98a84b } -.terminal-4179116690-r3 { fill: #9a9b99 } -.terminal-4179116690-r4 { fill: #608ab1 } -.terminal-4179116690-r5 { fill: #d0b344 } -.terminal-4179116690-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4179116690-r7 { fill: #868887 } + .terminal-4114956945-r1 { fill: #c5c8c6 } +.terminal-4114956945-r2 { fill: #98a84b } +.terminal-4114956945-r3 { fill: #9a9b99 } +.terminal-4114956945-r4 { fill: #608ab1 } +.terminal-4114956945-r5 { fill: #d0b344 } +.terminal-4114956945-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4114956945-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest┃ -┃Name       ┃Stars┃    Release┃    Released┃Last Pulled┃release?   ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ -│ rnafusion   │    76 │       2.1.0 │ 1 months ago │           - │ -           │ -│ smrnaseq    │    42 │       2.0.0 │ 3 months ago │           - │ -           │ -│ rnaseq      │   499 │       3.8.1 │ 3 months ago │           - │ -           │ -│ dualrnaseq  │     7 │       1.0.0 │  2 years ago │           - │ -           │ -│ circrna     │    19 │         dev │            - │           - │ -           │ -│ lncpipe     │    23 │         dev │            - │           - │ -           │ -│ scflow      │    12 │         dev │            - │           - │ -           │ -│ spatialtra… │    10 │         dev │            - │           - │ -           │ -└─────────────┴───────┴─────────────┴──────────────┴─────────────┴─────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest┃ +┃Name       ┃Stars┃    Release┃    Released┃Last Pulled┃release?   ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ +│ rnafusion   │    75 │       2.1.0 │ 1 months ago │           - │ -           │ +│ smrnaseq    │    42 │       2.0.0 │ 3 months ago │           - │ -           │ +│ rnaseq      │   499 │       3.8.1 │ 3 months ago │           - │ -           │ +│ dualrnaseq  │     7 │       1.0.0 │  2 years ago │           - │ -           │ +│ circrna     │    19 │         dev │            - │           - │ -           │ +│ lncpipe     │    23 │         dev │            - │           - │ -           │ +│ scflow      │    12 │         dev │            - │           - │ -           │ +│ spatialtra… │    10 │         dev │            - │           - │ -           │ +└─────────────┴───────┴─────────────┴──────────────┴─────────────┴─────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index cf4b23118e..8586699882 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-2036675850-matrix { + .terminal-1937061120-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2036675850-title { + .terminal-1937061120-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2036675850-r1 { fill: #c5c8c6 } -.terminal-2036675850-r2 { fill: #98a84b } -.terminal-2036675850-r3 { fill: #9a9b99 } -.terminal-2036675850-r4 { fill: #608ab1 } -.terminal-2036675850-r5 { fill: #d0b344 } -.terminal-2036675850-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2036675850-r7 { fill: #868887 } -.terminal-2036675850-r8 { fill: #868887;font-style: italic; } + .terminal-1937061120-r1 { fill: #c5c8c6 } +.terminal-1937061120-r2 { fill: #98a84b } +.terminal-1937061120-r3 { fill: #9a9b99 } +.terminal-1937061120-r4 { fill: #608ab1 } +.terminal-1937061120-r5 { fill: #d0b344 } +.terminal-1937061120-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1937061120-r7 { fill: #868887 } +.terminal-1937061120-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ rnaseq      │   499 │       3.8.1 │    3 months │           - │ -            │ -│             │       │             │         ago │             │              │ -│ sarek       │   186 │       3.0.1 │  6 days ago │           - │ -            │ -│ chipseq     │   121 │       1.2.2 │ 1 years ago │           - │ -            │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ +┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ +│ rnaseq      │   499 │       3.8.1 │    3 months │           - │ -            │ +│             │       │             │         ago │             │              │ +│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ +│ chipseq     │   121 │       1.2.2 │ 1 years ago │           - │ -            │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 5b0721e32e..4aa0ac54ac 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-2123714818-matrix { + .terminal-1152996364-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2123714818-title { + .terminal-1152996364-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2123714818-r1 { fill: #c5c8c6 } -.terminal-2123714818-r2 { fill: #98a84b } -.terminal-2123714818-r3 { fill: #9a9b99 } -.terminal-2123714818-r4 { fill: #608ab1 } -.terminal-2123714818-r5 { fill: #d0b344 } -.terminal-2123714818-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2123714818-r7 { fill: #868887 } -.terminal-2123714818-r8 { fill: #868887;font-style: italic; } + .terminal-1152996364-r1 { fill: #c5c8c6 } +.terminal-1152996364-r2 { fill: #98a84b } +.terminal-1152996364-r3 { fill: #9a9b99 } +.terminal-1152996364-r4 { fill: #608ab1 } +.terminal-1152996364-r5 { fill: #d0b344 } +.terminal-1152996364-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1152996364-r7 { fill: #868887 } +.terminal-1152996364-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ sarek       │   186 │       3.0.1 │  6 days ago │           - │ -            │ -│ epitopepre… │    22 │       2.1.0 │ 3 weeks ago │           - │ -            │ -│ eager       │    70 │       2.4.5 │ 3 weeks ago │           - │ -            │ -│ viralrecon  │    74 │         2.5 │    1 months │           - │ -            │ -│             │       │             │         ago │             │              │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ +┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ +│ mag         │    95 │       2.2.1 │ 5 hours ago │           - │ -            │ +│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ +│ epitopepre… │    22 │       2.1.0 │ 3 weeks ago │           - │ -            │ +│ eager       │    70 │       2.4.5 │ 3 weeks ago │           - │ -            │ +│ viralrecon  │    74 │         2.5 │    1 months │           - │ -            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index d0503aac4d..ccd9e9b873 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:195 -INFO     Linting module: 'multiqc'__init__.py:199 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ -│                                           ╷                ╷                 │ -│Module name                              │File path     │Test message   │ -│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ -│multiqc│modules/multi…│Conda update: │ -│││bioconda::mult…│ -│││1.10 -> 1.13a│ -│                                           ╵                ╵                 │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:190 +INFO     Linting module: 'multiqc'__init__.py:194 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ +│                                           ╷                ╷                 │ +│Module name                              │File path     │Test message   │ +│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ +│multiqc│modules/multi…│Conda update: │ +│││bioconda::mult…│ +│││1.10 -> 1.13a│ +│                                           ╵                ╵                 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index e53839201b..9297ec424b 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2388969700-matrix { + .terminal-1989200096-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2388969700-title { + .terminal-1989200096-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2388969700-r1 { fill: #c5c8c6 } -.terminal-2388969700-r2 { fill: #98a84b } -.terminal-2388969700-r3 { fill: #9a9b99 } -.terminal-2388969700-r4 { fill: #608ab1 } -.terminal-2388969700-r5 { fill: #d0b344 } -.terminal-2388969700-r6 { fill: #868887 } -.terminal-2388969700-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2388969700-r8 { fill: #868887;font-style: italic; } + .terminal-1989200096-r1 { fill: #c5c8c6 } +.terminal-1989200096-r2 { fill: #98a84b } +.terminal-1989200096-r3 { fill: #9a9b99 } +.terminal-1989200096-r4 { fill: #608ab1 } +.terminal-1989200096-r5 { fill: #d0b344 } +.terminal-1989200096-r6 { fill: #868887 } +.terminal-1989200096-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1989200096-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules installed in '.':                                   list.py:128 - -┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ -┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules installed in '.':                                   list.py:124 + +┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 328d5f100f..e26dcf52b8 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-1590927372-matrix { + .terminal-1871028241-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1590927372-title { + .terminal-1871028241-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1590927372-r1 { fill: #c5c8c6 } -.terminal-1590927372-r2 { fill: #98a84b } -.terminal-1590927372-r3 { fill: #9a9b99 } -.terminal-1590927372-r4 { fill: #608ab1 } -.terminal-1590927372-r5 { fill: #d0b344 } -.terminal-1590927372-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1590927372-r7 { fill: #868887 } -.terminal-1590927372-r8 { fill: #868887;font-style: italic; } + .terminal-1871028241-r1 { fill: #c5c8c6 } +.terminal-1871028241-r2 { fill: #98a84b } +.terminal-1871028241-r3 { fill: #9a9b99 } +.terminal-1871028241-r4 { fill: #608ab1 } +.terminal-1871028241-r5 { fill: #d0b344 } +.terminal-1871028241-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1871028241-r7 { fill: #868887 } +.terminal-1871028241-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):            list.py:123 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):            list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 2c9d755ab1..a8b3b7a17a 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-623533593-matrix { + .terminal-623664666-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-623533593-title { + .terminal-623664666-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-623533593-r1 { fill: #c5c8c6 } -.terminal-623533593-r2 { fill: #98a84b } -.terminal-623533593-r3 { fill: #9a9b99 } -.terminal-623533593-r4 { fill: #608ab1 } -.terminal-623533593-r5 { fill: #d0b344 } -.terminal-623533593-r6 { fill: #868887 } + .terminal-623664666-r1 { fill: #c5c8c6 } +.terminal-623664666-r2 { fill: #98a84b } +.terminal-623664666-r3 { fill: #9a9b99 } +.terminal-623664666-r4 { fill: #608ab1 } +.terminal-623664666-r5 { fill: #d0b344 } +.terminal-623664666-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Removing abacas                                            remove.py:51 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Removing abacas                                            remove.py:52 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 81f992bf71..57cfed4dd6 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Pipeline directory:                                          sync.py:95 -/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core-ne -xtbigthing -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 -INFO     Making a new template pipeline using pipeline variables     sync.py:223 + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Pipeline directory:                                          sync.py:95 +/home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 +INFO     Making a new template pipeline using pipeline variables     sync.py:223 From 8a2317b0afc9f0cd9d26e7765c75cde6db4fd341 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 25 Aug 2022 13:36:31 +0200 Subject: [PATCH 018/854] add option to remove git hosting from pipeline template --- nf_core/create.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 8e58306cd8..47d95f016c 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -58,6 +58,10 @@ def __init__( ) skippable_paths = { + "github": [ + ".github/", + ".gitignore", + ], "ci": [".github/workflows/"], "igenomes": ["conf/igenomes.config"], "branded": [ @@ -74,7 +78,7 @@ def __init__( self.name = self.template_params["name"] # Set fields used by the class methods - self.no_git = no_git + self.no_git = no_git if self.template_params["github"] else True self.force = force if outdir is None: outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) @@ -110,6 +114,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa # Define the different template areas, and what actions to take for each # if they are skipped template_areas = { + "github": {"name": "GitHub hosting", "file": True, "content": False}, "ci": {"name": "GitHub CI", "file": True, "content": False}, "github_badges": {"name": "GitHub badges", "file": False, "content": True}, "igenomes": {"name": "iGenomes config", "file": True, "content": True}, @@ -313,9 +318,10 @@ def render_template(self): # Make a logo and save it, if it is a nf-core pipeline self.make_pipeline_logo() else: - # Remove field mentioning nf-core docs - # in the github bug report template - self.remove_nf_core_in_bug_report_template() + if self.template_params["github"]: + # Remove field mentioning nf-core docs + # in the github bug report template + self.remove_nf_core_in_bug_report_template() # Update the .nf-core.yml with linting configurations self.fix_linting() From 1980d81e9582533d1d43b418e074f334a00ffbcf Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 25 Aug 2022 13:58:17 +0200 Subject: [PATCH 019/854] add file bug_report.yml to files_exist when github hosting is not added to the pipeline --- nf_core/create.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 47d95f016c..619a7b6cc8 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -78,7 +78,9 @@ def __init__( self.name = self.template_params["name"] # Set fields used by the class methods - self.no_git = no_git if self.template_params["github"] else True + self.no_git = ( + no_git if self.template_params["github"] else True + ) # Set to True if template was configured without github hosting self.force = force if outdir is None: outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) @@ -401,6 +403,15 @@ def fix_linting(self): "multiqc_config": ["report_comment"], } + # Add GitHub hosting specific configurations + if not self.template_params["github"]: + lint_config["files_exist"].extend( + [ + ".github/ISSUE_TEMPLATE/bug_report.yml", + ] + ) + lint_config["files_unchanged"] = [".github/ISSUE_TEMPLATE/bug_report.yml"] + # Add CI specific configurations if not self.template_params["ci"]: lint_config["files_exist"].extend( From 1e3efdec701a118b6de457232539b7ef5cf18286 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 25 Aug 2022 14:42:22 +0200 Subject: [PATCH 020/854] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5786f3038d..4a673609e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Make `nf-core create` fail if Git default branch name is dev or TEMPLATE ([#1705](https://github.com/nf-core/tools/pull/1705)) - Convert `console` snippets to `bash` snippets in the template where applicable ([#1729](https://github.com/nf-core/tools/pull/1729)) - Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728)) +- Add customisation option to remove all GitHub support with `nf-core create` ([#1766](https://github.com/nf-core/tools/pull/1766)) ### Linting From 0597be6b3cb1918f85e92ed3571b51adac6c4bd6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 29 Aug 2022 14:47:41 +0200 Subject: [PATCH 021/854] merge conflicts --- docs/images/nf-core-create.svg | 2 +- docs/images/nf-core-list-stars.svg | 4 ++-- docs/images/nf-core-list.svg | 4 ++-- docs/images/nf-core-modules-lint.svg | 2 +- docs/images/nf-core-modules-list-local.svg | 4 ++-- docs/images/nf-core-modules-list-remote.svg | 4 ++-- docs/images/nf-core-modules-remove.svg | 4 ++-- docs/images/nf-core-sync.svg | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index a453d8c57c..ddf6c0b299 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -131,7 +131,7 @@ - + diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index ce09b16f9d..650e88d007 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -111,9 +111,9 @@ - + - + $ nf-core list -s stars diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 3454cd6658..e846f83966 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -114,9 +114,9 @@ - + - + $ nf-core list diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index e74151121d..ad6d328bbb 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -148,7 +148,7 @@ - + diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 61c394913e..0df582bffb 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -132,9 +132,9 @@ - + - + $ nf-core modules list local diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 96b91a83f9..daf515dc51 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -132,9 +132,9 @@ - + - + $ nf-core modules list remote diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 1deb2efd38..1487fe7b16 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -88,9 +88,9 @@ - + - + $ nf-core modules remove abacas diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 1b6f3a9e83..1aa38f84b5 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -96,9 +96,9 @@ - + - + $ nf-core sync From 4ce9a753f59507d50b4006aa0f3ea2a215897f36 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 29 Aug 2022 12:59:11 +0000 Subject: [PATCH 022/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-create.svg | 217 ++++++++-------- docs/images/nf-core-list-rna.svg | 118 ++++----- docs/images/nf-core-list-stars.svg | 148 +++++------ docs/images/nf-core-list.svg | 154 ++++++------ docs/images/nf-core-modules-lint.svg | 260 +++++++++----------- docs/images/nf-core-modules-list-local.svg | 190 +++++++------- docs/images/nf-core-modules-list-remote.svg | 191 +++++++------- docs/images/nf-core-modules-remove.svg | 102 ++++---- docs/images/nf-core-sync.svg | 126 +++++----- 9 files changed, 721 insertions(+), 785 deletions(-) diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 37d5b1c38e..343fd47281 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - + + - + - - - - - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  -big omics technique" -a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:233 -INFO     Initialising pipeline git repository                      create.py:535 -INFO     Done. Remember to add a remote and push to GitHub:        create.py:542 - cd  -/home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin  -git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                    -INFO     This will also push your newly created dev branch and the create.py:548 -         TEMPLATE branch for syncing.                               -INFO    !!!!!! IMPORTANT !!!!!!create.py:224 - -If you are interested in adding your pipeline to the  -nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  -WRITING ANY CODE! - -Please read:  -https://nf-co.re/developers/adding_pipelines#join-the-com -munity + + + + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:233 +INFO     Initialising pipeline git repository                                          create.py:535 +INFO     Done. Remember to add a remote and push to GitHub:                            create.py:542 + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:548 +         syncing.                                                                       +INFO    !!!!!! IMPORTANT !!!!!!create.py:224 + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 93df06fcb5..cbd098b947 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-4158313489-matrix { + .terminal-4236301330-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4158313489-title { + .terminal-4236301330-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4158313489-r1 { fill: #c5c8c6 } -.terminal-4158313489-r2 { fill: #98a84b } -.terminal-4158313489-r3 { fill: #9a9b99 } -.terminal-4158313489-r4 { fill: #608ab1 } -.terminal-4158313489-r5 { fill: #d0b344 } -.terminal-4158313489-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4158313489-r7 { fill: #868887 } + .terminal-4236301330-r1 { fill: #c5c8c6 } +.terminal-4236301330-r2 { fill: #98a84b } +.terminal-4236301330-r3 { fill: #9a9b99 } +.terminal-4236301330-r4 { fill: #608ab1 } +.terminal-4236301330-r5 { fill: #d0b344 } +.terminal-4236301330-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4236301330-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnafusion            │    75 │          2.1.0 │ 2 months ago │           - │ -                   │ -│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ -│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ -│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    19 │            dev │            - │           - │ -                   │ -│ lncpipe              │    23 │            dev │            - │           - │ -                   │ -│ scflow               │    12 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ +│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    19 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    12 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 1304dc5018..da3f02c7db 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,89 +19,89 @@ font-weight: 700; } - .terminal-1937061120-matrix { + .terminal-1473353797-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1937061120-title { + .terminal-1473353797-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1937061120-r1 { fill: #c5c8c6 } -.terminal-1937061120-r2 { fill: #98a84b } -.terminal-1937061120-r3 { fill: #9a9b99 } -.terminal-1937061120-r4 { fill: #608ab1 } -.terminal-1937061120-r5 { fill: #d0b344 } -.terminal-1937061120-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1937061120-r7 { fill: #868887 } -.terminal-1937061120-r8 { fill: #868887;font-style: italic; } + .terminal-1473353797-r1 { fill: #c5c8c6 } +.terminal-1473353797-r2 { fill: #98a84b } +.terminal-1473353797-r3 { fill: #9a9b99 } +.terminal-1473353797-r4 { fill: #608ab1 } +.terminal-1473353797-r5 { fill: #d0b344 } +.terminal-1473353797-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1473353797-r7 { fill: #868887 } +.terminal-1473353797-r8 { fill: #868887;font-style: italic; } - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -111,30 +111,30 @@ - - - - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ rnaseq      │   499 │       3.8.1 │    3 months │           - │ -            │ -│             │       │             │         ago │             │              │ -│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ -│ chipseq     │   121 │       1.2.2 │ 1 years ago │           - │ -            │ -[..truncated..] + + + + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ chipseq              │   121 │          1.2.2 │  1 years ago │           - │ -                   │ +│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 05f3d7c111..c5e7e17dd8 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,92 +19,92 @@ font-weight: 700; } - .terminal-1065112585-matrix { + .terminal-2711598360-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1065112585-title { + .terminal-2711598360-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1065112585-r1 { fill: #c5c8c6 } -.terminal-1065112585-r2 { fill: #98a84b } -.terminal-1065112585-r3 { fill: #9a9b99 } -.terminal-1065112585-r4 { fill: #608ab1 } -.terminal-1065112585-r5 { fill: #d0b344 } -.terminal-1065112585-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1065112585-r7 { fill: #868887 } -.terminal-1065112585-r8 { fill: #868887;font-style: italic; } + .terminal-2711598360-r1 { fill: #c5c8c6 } +.terminal-2711598360-r2 { fill: #98a84b } +.terminal-2711598360-r3 { fill: #9a9b99 } +.terminal-2711598360-r4 { fill: #608ab1 } +.terminal-2711598360-r5 { fill: #d0b344 } +.terminal-2711598360-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2711598360-r7 { fill: #868887 } +.terminal-2711598360-r8 { fill: #868887;font-style: italic; } - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -114,31 +114,31 @@ - - - - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ mag         │    95 │       2.2.1 │ 2 hours ago │           - │ -            │ -│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ -│ epitopepre… │    22 │       2.1.0 │ 3 weeks ago │           - │ -            │ -│ eager       │    70 │       2.4.5 │ 3 weeks ago │           - │ -            │ -│ viralrecon  │    74 │         2.5 │    1 months │           - │ -            │ -[..truncated..] + + + + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ mag                  │    99 │          2.2.1 │   4 days ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ +│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ +│ viralrecon           │    74 │            2.5 │ 2 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 9354a6d3ff..e74151121d 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:195 -INFO     Linting module: 'multiqc'__init__.py:199 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ -│                                           ╷                ╷                 │ -│Module name                              │File path     │Test message   │ -│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ -│multiqc│modules/multi…│Conda update: │ -│││bioconda::mult…│ -│││1.10 -> 1.13a│ -│                                           ╵                ╵                 │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:206 +INFO     Linting module: 'multiqc'__init__.py:210 + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ +│                                           ╷                         ╷                            │ +│Module name                              │File path              │Test message              │ +│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ +│multiqc│modules/multiqc/main.nf│Conda update: │ +│││bioconda::multiqc 1.10 -> │ +│││1.13a│ +│                                           ╵                         ╵                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index b61893e560..61c394913e 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,110 +19,110 @@ font-weight: 700; } - .terminal-1989200096-matrix { + .terminal-578959072-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1989200096-title { + .terminal-578959072-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1989200096-r1 { fill: #c5c8c6 } -.terminal-1989200096-r2 { fill: #98a84b } -.terminal-1989200096-r3 { fill: #9a9b99 } -.terminal-1989200096-r4 { fill: #608ab1 } -.terminal-1989200096-r5 { fill: #d0b344 } -.terminal-1989200096-r6 { fill: #868887 } -.terminal-1989200096-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-1989200096-r8 { fill: #868887;font-style: italic; } + .terminal-578959072-r1 { fill: #c5c8c6 } +.terminal-578959072-r2 { fill: #98a84b } +.terminal-578959072-r3 { fill: #9a9b99 } +.terminal-578959072-r4 { fill: #608ab1 } +.terminal-578959072-r5 { fill: #d0b344 } +.terminal-578959072-r6 { fill: #868887 } +.terminal-578959072-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-578959072-r8 { fill: #868887;font-style: italic; } - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -132,37 +132,37 @@ - - - - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules installed in '.':                                   list.py:124 - -┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ -┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -[..truncated..] + + + + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                       list.py:124 + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name         ┃Repository     ┃Version SHA        ┃Message             ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftware… │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ fastqc               │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ multiqc              │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 1d49d8ba10..96b91a83f9 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,110 +19,110 @@ font-weight: 700; } - .terminal-1871028241-matrix { + .terminal-277652753-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1871028241-title { + .terminal-277652753-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1871028241-r1 { fill: #c5c8c6 } -.terminal-1871028241-r2 { fill: #98a84b } -.terminal-1871028241-r3 { fill: #9a9b99 } -.terminal-1871028241-r4 { fill: #608ab1 } -.terminal-1871028241-r5 { fill: #d0b344 } -.terminal-1871028241-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1871028241-r7 { fill: #868887 } -.terminal-1871028241-r8 { fill: #868887;font-style: italic; } + .terminal-277652753-r1 { fill: #c5c8c6 } +.terminal-277652753-r2 { fill: #98a84b } +.terminal-277652753-r3 { fill: #9a9b99 } +.terminal-277652753-r4 { fill: #608ab1 } +.terminal-277652753-r5 { fill: #d0b344 } +.terminal-277652753-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-277652753-r7 { fill: #868887 } +.terminal-277652753-r8 { fill: #868887;font-style: italic; } - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -132,38 +132,37 @@ - - - - - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):            list.py:119 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + + + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):                                list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 477e403a8c..1deb2efd38 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,66 +19,66 @@ font-weight: 700; } - .terminal-623664666-matrix { + .terminal-1679646874-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-623664666-title { + .terminal-1679646874-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-623664666-r1 { fill: #c5c8c6 } -.terminal-623664666-r2 { fill: #98a84b } -.terminal-623664666-r3 { fill: #9a9b99 } -.terminal-623664666-r4 { fill: #608ab1 } -.terminal-623664666-r5 { fill: #d0b344 } -.terminal-623664666-r6 { fill: #868887 } + .terminal-1679646874-r1 { fill: #c5c8c6 } +.terminal-1679646874-r2 { fill: #98a84b } +.terminal-1679646874-r3 { fill: #9a9b99 } +.terminal-1679646874-r4 { fill: #608ab1 } +.terminal-1679646874-r5 { fill: #d0b344 } +.terminal-1679646874-r6 { fill: #868887 } - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -88,23 +88,23 @@ - - - - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Removing abacas                                            remove.py:52 + + + + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Removing abacas                                                                remove.py:52 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index c39d58fef1..1b6f3a9e83 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - + + - + - - - - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Pipeline directory:                                          sync.py:95 -/home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 -INFO     Making a new template pipeline using pipeline variables     sync.py:223 + + + + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 +INFO     Making a new template pipeline using pipeline variables                         sync.py:223 From 765c04e7f7f0c6a097a68144966c78f29599cc51 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 29 Aug 2022 15:37:44 +0200 Subject: [PATCH 023/854] add documentation for excluding GitHub during pipeline creation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7703a91c83..859eaf8b24 100644 --- a/README.md +++ b/README.md @@ -406,13 +406,14 @@ description: A cool pipeline author: me prefix: cool-pipes-company skip: + - github - ci - github_badges - igenomes - nf_core_configs ``` -This will create a pipeline called `cool-pipe` in the directory `cool-pipes-company-cool-pipe` with `me` as the author. It will exclude the GitHub CI from the pipeline, remove GitHub badges from the `README.md` file, remove pipeline options related to iGenomes and exclude `nf_core/configs` options. +This will create a pipeline called `cool-pipe` in the directory `cool-pipes-company-cool-pipe` with `me` as the author. It will exclude all files required for GitHub hosting of the pipeline, the GitHub CI from the pipeline, remove GitHub badges from the `README.md` file and remove pipeline options related to iGenomes and exclude `nf_core/configs` options. To run the pipeline creation silently (i.e. without any prompts) with the nf-core template, you can use the `--plain` option. From 2ada5b3e9582008cdc54af49de293480816b6555 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 29 Aug 2022 15:43:30 +0200 Subject: [PATCH 024/854] remove and --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 859eaf8b24..7658608fc0 100644 --- a/README.md +++ b/README.md @@ -413,7 +413,7 @@ skip: - nf_core_configs ``` -This will create a pipeline called `cool-pipe` in the directory `cool-pipes-company-cool-pipe` with `me` as the author. It will exclude all files required for GitHub hosting of the pipeline, the GitHub CI from the pipeline, remove GitHub badges from the `README.md` file and remove pipeline options related to iGenomes and exclude `nf_core/configs` options. +This will create a pipeline called `cool-pipe` in the directory `cool-pipes-company-cool-pipe` with `me` as the author. It will exclude all files required for GitHub hosting of the pipeline, the GitHub CI from the pipeline, remove GitHub badges from the `README.md` file, remove pipeline options related to iGenomes and exclude `nf_core/configs` options. To run the pipeline creation silently (i.e. without any prompts) with the nf-core template, you can use the `--plain` option. From 9402364e823217b1a65daa29bd5844a101cfd4d6 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 25 Aug 2022 16:25:38 +0200 Subject: [PATCH 025/854] disable unnecessary pylint warnings --- nf_core/__main__.py | 4 ++-- nf_core/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0746e80c0e..16aa571998 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -428,7 +428,7 @@ def remote(ctx, keywords, json): default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def local(ctx, keywords, json, dir): +def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List modules installed locally in a pipeline """ @@ -680,7 +680,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("--passed", is_flag=True, help="Show passed tests") @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") @common_options -def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version, hide_progress): +def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version, hide_progress): # pylint: disable=redefined-outer-name """ Lint one or more modules in a directory. diff --git a/nf_core/utils.py b/nf_core/utils.py index 1c65e45bb8..9321ff9629 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -409,7 +409,7 @@ class GitHub_API_Session(requests_cache.CachedSession): such as automatically setting up GitHub authentication if we can. """ - def __init__(self): + def __init__(self): # pylint: disable=super-init-not-called self.auth_mode = None self.return_ok = [200, 201] self.return_retry = [403] From 2c711252ab9a27ac03f6b9a77a9eeda78c08a9d5 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 25 Aug 2022 16:27:33 +0200 Subject: [PATCH 026/854] remove unnecessary f-strings --- nf_core/modules/update.py | 4 ++-- nf_core/schema.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 4644c2398e..124baccfb5 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -178,9 +178,9 @@ def update(self, module=None): if dry_run: if patch_relpath is not None: if patch_successful: - log.info(f"Current installation is compared against patched version in remote") + log.info("Current installation is compared against patched version in remote.") else: - log.warning(f"Current installation is compared against unpatched version in remote") + log.warning("Current installation is compared against unpatched version in remote.") # Compute the diffs for the module if self.save_diff_fn: log.info(f"Writing diff file for module '{module_fullname}' to '{self.save_diff_fn}'") diff --git a/nf_core/schema.py b/nf_core/schema.py index 1dab8c2b0e..6804183cb8 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -434,10 +434,10 @@ def check_for_input_mimetype(self): # Check that the input parameter is defined if "input" not in self.schema_params: - raise LookupError(f"Parameter `input` not found in schema") + raise LookupError("Parameter `input` not found in schema") # Check that the input parameter is defined in the right place if "input" not in self.schema.get("definitions", {}).get("input_output_options", {}).get("properties", {}): - raise LookupError(f"Parameter `input` is not defined in the correct subschema (input_output_options)") + raise LookupError("Parameter `input` is not defined in the correct subschema (input_output_options)") input_entry = self.schema["definitions"]["input_output_options"]["properties"]["input"] if "mimetype" not in input_entry: return None From 90cc6ff10bb254f257a1a90460d3f88c329e2394 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 25 Aug 2022 16:27:44 +0200 Subject: [PATCH 027/854] use super init instead of repeated code --- nf_core/modules/lint/__init__.py | 8 +------- nf_core/modules/modules_command.py | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 32bfa8ce95..dd1fd5b7d2 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -73,18 +73,12 @@ def __init__( no_pull=False, hide_progress=False, ): - self.dir = dir - try: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) - except LookupError as e: - raise UserWarning(e) + super().__init__(dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=False) self.fail_warned = fail_warned self.passed = [] self.warned = [] self.failed = [] - self.hide_progress = hide_progress - self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.lint_tests = self.get_all_lint_tests(self.repo_type == "pipeline") if self.repo_type == "pipeline": diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index dbfd1ff6dd..01d3e7e2f3 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -19,11 +19,11 @@ class ModuleCommand: Base class for the 'nf-core modules' commands """ - def __init__(self, dir, remote_url=None, branch=None, no_pull=False): + def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): """ Initialise the ModulesCommand object """ - self.modules_repo = ModulesRepo(remote_url, branch, no_pull) + self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.dir = dir try: if self.dir: From bf498a3902e172fea8bc1d8234fbe7e65c24a1a9 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 29 Aug 2022 16:03:12 +0200 Subject: [PATCH 028/854] reinstate hide_progress as a property --- nf_core/modules/modules_command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 01d3e7e2f3..7a24183271 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -24,6 +24,7 @@ def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progre Initialise the ModulesCommand object """ self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) + self.hide_progress = hide_progress self.dir = dir try: if self.dir: From 73f7420c5659d610768e72de9e93b8a906392ef0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 29 Aug 2022 14:06:40 +0000 Subject: [PATCH 029/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-create.svg | 214 +++++++-------- docs/images/nf-core-lint.svg | 264 +++++++++++------- docs/images/nf-core-list-rna.svg | 170 ++++++------ docs/images/nf-core-list-stars.svg | 148 +++++----- docs/images/nf-core-list.svg | 154 +++++------ docs/images/nf-core-modules-lint.svg | 284 ++++++++++---------- docs/images/nf-core-modules-list-local.svg | 190 ++++++------- docs/images/nf-core-modules-list-remote.svg | 190 ++++++------- docs/images/nf-core-modules-mulled.svg | 80 +++--- docs/images/nf-core-modules-remove.svg | 102 +++---- docs/images/nf-core-sync.svg | 122 ++++----- 11 files changed, 980 insertions(+), 938 deletions(-) diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 6b6667a5e9..343fd47281 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - + + - + - - - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  -big omics technique" -a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:226 -INFO     Initialising pipeline git repository                      create.py:518 -INFO     Done. Remember to add a remote and push to GitHub:        create.py:525 - cd  -/home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin  -git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                    -INFO     This will also push your newly created dev branch and the create.py:531 -         TEMPLATE branch for syncing.                               -INFO    !!!!!! IMPORTANT !!!!!!create.py:217 - -If you are interested in adding your pipeline to the  -nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  -WRITING ANY CODE! - -Please read:  -https://nf-co.re/developers/adding_pipelines#join-the-com -munity + + + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:233 +INFO     Initialising pipeline git repository                                          create.py:535 +INFO     Done. Remember to add a remote and push to GitHub:                            create.py:542 + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:548 +         syncing.                                                                       +INFO    !!!!!! IMPORTANT !!!!!!create.py:224 + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 03d39778ef..1f4b387b45 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Testing pipeline: .__init__.py:263 - - -╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/modules…│New version available│ -│fastqc│modules/nf-core/modules…│New version available│ -│multiqc│modules/nf-core/modules…│New version available│ -│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ -│samplesheet_check│modules/local/sampleshe…│when: condition has been │ -│││removed│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 169 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   5 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Testing pipeline: .__init__.py:263 + + +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check + + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc + + +╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/modules…│New version available│ +│fastqc│modules/nf-core/modules…│New version available│ +│multiqc│modules/nf-core/modules…│New version available│ +│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ +│samplesheet_check│modules/local/sampleshe…│when: condition has been │ +│││removed│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 169 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   5 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index e893848c09..cbd098b947 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest┃ -┃Name       ┃Stars┃    Release┃    Released┃Last Pulled┃release?   ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ -│ rnafusion   │    75 │       2.1.0 │ 1 months ago │           - │ -           │ -│ smrnaseq    │    42 │       2.0.0 │ 3 months ago │           - │ -           │ -│ rnaseq      │   499 │       3.8.1 │ 3 months ago │           - │ -           │ -│ dualrnaseq  │     7 │       1.0.0 │  2 years ago │           - │ -           │ -│ circrna     │    19 │         dev │            - │           - │ -           │ -│ lncpipe     │    23 │         dev │            - │           - │ -           │ -│ scflow      │    12 │         dev │            - │           - │ -           │ -│ spatialtra… │    10 │         dev │            - │           - │ -           │ -└─────────────┴───────┴─────────────┴──────────────┴─────────────┴─────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ +│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    19 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    12 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 8586699882..da3f02c7db 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ rnaseq      │   499 │       3.8.1 │    3 months │           - │ -            │ -│             │       │             │         ago │             │              │ -│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ -│ chipseq     │   121 │       1.2.2 │ 1 years ago │           - │ -            │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ chipseq              │   121 │          1.2.2 │  1 years ago │           - │ -                   │ +│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 4aa0ac54ac..c5e7e17dd8 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ mag         │    95 │       2.2.1 │ 5 hours ago │           - │ -            │ -│ sarek       │   186 │       3.0.1 │  1 week ago │           - │ -            │ -│ epitopepre… │    22 │       2.1.0 │ 3 weeks ago │           - │ -            │ -│ eager       │    70 │       2.4.5 │ 3 weeks ago │           - │ -            │ -│ viralrecon  │    74 │         2.5 │    1 months │           - │ -            │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ mag                  │    99 │          2.2.1 │   4 days ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ +│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ +│ viralrecon           │    74 │            2.5 │ 2 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index ccd9e9b873..ddd4480bb7 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:190 -INFO     Linting module: 'multiqc'__init__.py:194 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ -│                                           ╷                ╷                 │ -│Module name                              │File path     │Test message   │ -│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ -│multiqc│modules/multi…│Conda update: │ -│││bioconda::mult…│ -│││1.10 -> 1.13a│ -│                                           ╵                ╵                 │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:200 +INFO     Linting module: 'multiqc'__init__.py:204 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ +│                                           ╷                         ╷                            │ +│Module name                              │File path              │Test message              │ +│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ +│multiqc│modules/multiqc/main.nf│Conda update: │ +│││bioconda::multiqc 1.10 -> │ +│││1.13a│ +│                                           ╵                         ╵                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 9297ec424b..61c394913e 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules installed in '.':                                   list.py:124 - -┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ -┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                       list.py:124 + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name         ┃Repository     ┃Version SHA        ┃Message             ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftware… │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ fastqc               │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ multiqc              │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index e26dcf52b8..96b91a83f9 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):            list.py:119 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):                                list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index b7ac11336c..4e13039d6b 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -19,70 +19,70 @@ font-weight: 700; } - .terminal-3375078311-matrix { + .terminal-3387661225-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3375078311-title { + .terminal-3387661225-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3375078311-r1 { fill: #c5c8c6 } -.terminal-3375078311-r2 { fill: #98a84b } -.terminal-3375078311-r3 { fill: #9a9b99 } -.terminal-3375078311-r4 { fill: #608ab1 } -.terminal-3375078311-r5 { fill: #d0b344 } -.terminal-3375078311-r6 { fill: #868887 } -.terminal-3375078311-r7 { fill: #00823d;font-weight: bold } -.terminal-3375078311-r8 { fill: #68a0b3;font-weight: bold } + .terminal-3387661225-r1 { fill: #c5c8c6 } +.terminal-3387661225-r2 { fill: #98a84b } +.terminal-3387661225-r3 { fill: #9a9b99 } +.terminal-3387661225-r4 { fill: #608ab1 } +.terminal-3387661225-r5 { fill: #d0b344 } +.terminal-3387661225-r6 { fill: #868887 } +.terminal-3387661225-r7 { fill: #00823d;font-weight: bold } +.terminal-3387661225-r8 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -94,23 +94,23 @@ - + - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                              mulled.py:68 -INFO     Mulled container hash:                                                      __main__.py:826 -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                              mulled.py:68 +INFO     Mulled container hash:                                                      __main__.py:828 +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index a8b3b7a17a..1deb2efd38 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Removing abacas                                            remove.py:52 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Removing abacas                                                                remove.py:52 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 57cfed4dd6..1b6f3a9e83 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - + + - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Pipeline directory:                                          sync.py:95 -/home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 -INFO     Making a new template pipeline using pipeline variables     sync.py:223 + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 +INFO     Making a new template pipeline using pipeline variables                         sync.py:223 From a11e07e57e34447ff4c2a5acf6ebf52dd4224173 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 29 Aug 2022 16:27:49 +0200 Subject: [PATCH 030/854] exclude github_badges if github hosting is excluded --- nf_core/create.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 619a7b6cc8..1cd4fecba0 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -146,6 +146,9 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict[t_area] = False else: param_dict[t_area] = True + # If github is selected, exclude also github_badges + if not param_dict["github"]: + param_dict["github_badges"] = False # Set the last parameters based on the ones provided param_dict["short_name"] = ( @@ -436,7 +439,7 @@ def fix_linting(self): ) # Add github badges specific configurations - if not self.template_params["github_badges"]: + if not self.template_params["github_badges"] or not self.template_params["github"]: lint_config["readme"] = ["nextflow_badge"] # Add the lint content to the preexisting nf-core config From 0518152a8caf3d842250d9e7e949f780ec9ce3f1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 29 Aug 2022 14:31:20 +0000 Subject: [PATCH 031/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-create.svg | 132 ++++++++++++++++----------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 343fd47281..2158fb082e 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-1691595264-matrix { + .terminal-2228597254-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1691595264-title { + .terminal-2228597254-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1691595264-r1 { fill: #c5c8c6 } -.terminal-1691595264-r2 { fill: #98a84b } -.terminal-1691595264-r3 { fill: #9a9b99 } -.terminal-1691595264-r4 { fill: #608ab1 } -.terminal-1691595264-r5 { fill: #d0b344 } -.terminal-1691595264-r6 { fill: #868887 } -.terminal-1691595264-r7 { fill: #98729f } -.terminal-1691595264-r8 { fill: #ff2c7a } -.terminal-1691595264-r9 { fill: #98a84b;font-weight: bold } -.terminal-1691595264-r10 { fill: #1984e9;text-decoration: underline; } + .terminal-2228597254-r1 { fill: #c5c8c6 } +.terminal-2228597254-r2 { fill: #98a84b } +.terminal-2228597254-r3 { fill: #9a9b99 } +.terminal-2228597254-r4 { fill: #608ab1 } +.terminal-2228597254-r5 { fill: #d0b344 } +.terminal-2228597254-r6 { fill: #868887 } +.terminal-2228597254-r7 { fill: #98729f } +.terminal-2228597254-r8 { fill: #ff2c7a } +.terminal-2228597254-r9 { fill: #98a84b;font-weight: bold } +.terminal-2228597254-r10 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,35 +132,35 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:233 -INFO     Initialising pipeline git repository                                          create.py:535 -INFO     Done. Remember to add a remote and push to GitHub:                            create.py:542 - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:548 -         syncing.                                                                       -INFO    !!!!!! IMPORTANT !!!!!!create.py:224 - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 +INFO     Initialising pipeline git repository                                          create.py:538 +INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 +         syncing.                                                                       +INFO    !!!!!! IMPORTANT !!!!!!create.py:227 + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community From d6ba4da6c2603b04846cba8171ea78310108312b Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 29 Aug 2022 18:18:01 +0200 Subject: [PATCH 032/854] fix anomalous backslash --- nf_core/refgenie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py index 6221efce68..e8d421d176 100644 --- a/nf_core/refgenie.py +++ b/nf_core/refgenie.py @@ -83,7 +83,7 @@ def _update_nextflow_home_config(refgenie_genomes_config_file, nxf_home): with open(nxf_home_config, "r") as fh: lines = fh.readlines() for line in lines: - if re.match(f"\s*includeConfig\s*'{os.path.abspath(refgenie_genomes_config_file)}'", line): + if re.match(rf"\s*includeConfig\s*'{os.path.abspath(refgenie_genomes_config_file)}'", line): has_include_statement = True break From b3e4e948ffe46bbaa74b6f58c3a6163bc849103d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 29 Aug 2022 16:26:37 +0000 Subject: [PATCH 033/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-modules-create.svg | 130 +++++------- docs/images/nf-core-modules-info.svg | 226 +++++++++++---------- docs/images/nf-core-modules-install.svg | 88 ++++---- docs/images/nf-core-modules-lint.svg | 208 ++++++++++--------- docs/images/nf-core-modules-list-local.svg | 137 ++++--------- docs/images/nf-core-modules-patch.svg | 150 ++++---------- docs/images/nf-core-modules-remove.svg | 75 +++---- docs/images/nf-core-modules-update.svg | 95 ++++----- 8 files changed, 471 insertions(+), 638 deletions(-) diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 13e19b5e7a..47be3b89c3 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Repository type: modulescreate.py:93 -INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 -responses. ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 -INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 -INFO     Using Singularity container:                                                  create.py:192 -'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' -INFO     Created / edited following files:                                             create.py:270 -           ./modules/fastqc/main.nf -           ./modules/fastqc/meta.yml -           ./tests/modules/fastqc/main.nf -           ./tests/modules/fastqc/test.yml -           ./tests/modules/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + +INFO     Repository type: modulescreate.py:93 +INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 +responses. ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 +INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 +INFO     Using Singularity container:                                                  create.py:192 +'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 5c3a82569d..9659c49c45 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - - - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Reinstalling modules found in 'modules.json' but missing from           modules_json.py:452 +         directory: 'nf-core/modules/custom/dumpsoftwareversions',                +'nf-core/modules/fastqc', 'nf-core/modules/multiqc' +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index ce016fe92b..3b8fc97194 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Installing 'abacas'install.py:116 -INFO     Include statement: include { ABACAS } from                                   install.py:125 -'../modules/nf-core/modules/abacas/main' + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:483 diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index ddd4480bb7..24ea8054b9 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:200 -INFO     Linting module: 'multiqc'__init__.py:204 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ -│                                           ╷                         ╷                            │ -│Module name                              │File path              │Test message              │ -│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ -│multiqc│modules/multiqc/main.nf│Conda update: │ -│││bioconda::multiqc 1.10 -> │ -│││1.13a│ -│                                           ╵                         ╵                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:200 +INFO     Linting module: 'multiqc'__init__.py:204 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ +│                                           ╷                         ╷                            │ +│Module name                              │File path              │Test message              │ +│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ +│multiqc│modules/multiqc/main.nf│Conda update: │ +│││bioconda::multiqc 1.10 -> │ +│││1.13a│ +│                                           ╵                         ╵                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 61c394913e..a8957f7e99 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                       list.py:124 - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name         ┃Repository     ┃Version SHA        ┃Message             ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftware… │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -│                      │                 │                     │ yaml files, add      │            │ -│                      │                 │                     │ yamllint config      │            │ -│                      │                 │                     │ (#1279)              │            │ -│ fastqc               │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -│                      │                 │                     │ yaml files, add      │            │ -│                      │                 │                     │ yamllint config      │            │ -│                      │                 │                     │ (#1279)              │            │ -│ multiqc              │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'list.py:64 + diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 8dea1566a8..757ef4abf9 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Changes in module 'nf-core/modules/fastqc'modules_differ.py:252 -INFO    'modules/nf-core/modules/fastqc/meta.yml' is unchanged                modules_differ.py:257 -INFO     Changes in 'fastqc/main.nf':                                          modules_differ.py:266 - ---- modules/nf-core/modules/fastqc/main.nf -+++ modules/nf-core/modules/fastqc/main.nf -@@ -1,6 +1,6 @@ -process FASTQC {                                                                                   -    tag "$meta.id"                                                                                 --    label 'process_medium' -+    label 'process_low' - -    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)                                 -    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  - - -INFO     Patch file of 'nf-core/modules/fastqc' written to                              patch.py:115 -'modules/nf-core/modules/fastqc/fastqc.diff' + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:571 diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 1deb2efd38..20726d93e6 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,65 +19,66 @@ font-weight: 700; } - .terminal-1679646874-matrix { + .terminal-1917479430-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1679646874-title { + .terminal-1917479430-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1679646874-r1 { fill: #c5c8c6 } -.terminal-1679646874-r2 { fill: #98a84b } -.terminal-1679646874-r3 { fill: #9a9b99 } -.terminal-1679646874-r4 { fill: #608ab1 } -.terminal-1679646874-r5 { fill: #d0b344 } -.terminal-1679646874-r6 { fill: #868887 } + .terminal-1917479430-r1 { fill: #c5c8c6 } +.terminal-1917479430-r2 { fill: #98a84b } +.terminal-1917479430-r3 { fill: #9a9b99 } +.terminal-1917479430-r4 { fill: #608ab1 } +.terminal-1917479430-r5 { fill: #d0b344 } +.terminal-1917479430-r6 { fill: #292929;font-weight: bold } +.terminal-1917479430-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +90,22 @@ - - - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Removing abacas                                                                remove.py:52 + + + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +CRITICAL Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:599 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 20db553630..2acd0bdba6 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO    'nf-core/modules/abacas' is already up to date                                update.py:160 -INFO     Updating 'nf-core/modules/custom/dumpsoftwareversions'update.py:516 -INFO     Updating 'nf-core/modules/fastqc'update.py:516 -INFO     Updating 'nf-core/modules/multiqc'update.py:516 -INFO     Updates complete ✨                                                           update.py:242 + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re + + + +ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:540 From 47554cc744d705f8983d5049b978e9184b7b4bc8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Aug 2022 13:45:50 +0200 Subject: [PATCH 034/854] change-rich-codex-run-condition --- .github/workflows/rich-codex.yml | 7 +++---- README.md | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index d2894e9dad..a85fea129d 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -1,12 +1,11 @@ name: Generate images for docs on: - push: - branches-ignore: - - "master" - - "dev" + pull_request_review: + types: [submitted] workflow_dispatch: jobs: rich_codex: + if: github.event.review.state == 'approved' runs-on: ubuntu-latest steps: - name: Check out the repo diff --git a/README.md b/README.md index 7703a91c83..5fefc63475 100644 --- a/README.md +++ b/README.md @@ -813,6 +813,8 @@ After you have written a minimal Nextflow script to test your module `modules/te ![`nf-core modules create-test-yml fastqc --no-prompts --force`](docs/images/nf-core-modules-create-test.svg) From f0058f3a0b3de528654ad94e2f4d8048958deab0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Aug 2022 14:11:46 +0200 Subject: [PATCH 035/854] remove change thresholds --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5fefc63475..1b30c93f08 100644 --- a/README.md +++ b/README.md @@ -424,7 +424,6 @@ This is the same test that is used on the automated continuous integration tests For example, the current version looks something like this: From 117a006915fa19c9d2a7096159e371c63f16e06b Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Aug 2022 14:28:50 +0200 Subject: [PATCH 036/854] add PR ref --- .github/workflows/rich-codex.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index a85fea129d..ec364291c6 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -10,7 +10,8 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v3 - + with: + ref: ${{ github.event.pull_request.head.ref }} - name: Set up Python uses: actions/setup-python@v3 with: From 4d24ba296546b85dcf208844cbf9d9017383a599 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Aug 2022 14:32:39 +0200 Subject: [PATCH 037/854] fix typo --- .github/workflows/rich-codex.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index ec364291c6..a2d3ddc9bc 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -11,7 +11,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v3 with: - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head_ref }} - name: Set up Python uses: actions/setup-python@v3 with: From cdd7b58a868d8f0da231138ebad6918de3f719e3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Aug 2022 14:39:56 +0200 Subject: [PATCH 038/854] forget it, we are doing it manually --- .github/workflows/rich-codex.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index a2d3ddc9bc..669c138f46 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -1,17 +1,12 @@ name: Generate images for docs on: - pull_request_review: - types: [submitted] workflow_dispatch: jobs: rich_codex: - if: github.event.review.state == 'approved' runs-on: ubuntu-latest steps: - name: Check out the repo uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head_ref }} - name: Set up Python uses: actions/setup-python@v3 with: From c5ddf28b1b7849a427f1f07d646a6e9a8f11e969 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Aug 2022 15:05:02 +0200 Subject: [PATCH 039/854] Bump to v2.5 for release --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a673609e6..aa177f6ecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v2.5dev +## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] ### Template diff --git a/setup.py b/setup.py index f24b87001e..ac95a82c90 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.5dev" +version = "2.5" with open("README.md") as f: readme = f.read() From cd0ac0b5ee826304a0fd77792593735dd8fc2e58 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 30 Aug 2022 13:25:07 +0000 Subject: [PATCH 040/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 90 +++---- docs/images/nf-core-create.svg | 132 ++++----- docs/images/nf-core-download.svg | 88 +++--- docs/images/nf-core-launch-rnaseq.svg | 84 +++--- docs/images/nf-core-licences.svg | 132 ++++----- docs/images/nf-core-lint.svg | 270 ++++++++++--------- docs/images/nf-core-list-rna.svg | 118 ++++---- docs/images/nf-core-list-stars.svg | 104 +++---- docs/images/nf-core-list.svg | 108 ++++---- docs/images/nf-core-modules-bump-version.svg | 106 ++++---- docs/images/nf-core-modules-create-test.svg | 126 ++++----- docs/images/nf-core-modules-create.svg | 130 +++++---- docs/images/nf-core-modules-info.svg | 226 ++++++++-------- docs/images/nf-core-modules-install.svg | 88 +++--- docs/images/nf-core-modules-lint.svg | 204 +++++++------- docs/images/nf-core-modules-list-local.svg | 137 +++++++--- docs/images/nf-core-modules-list-remote.svg | 132 ++++----- docs/images/nf-core-modules-mulled.svg | 80 +++--- docs/images/nf-core-modules-patch.svg | 150 ++++++++--- docs/images/nf-core-modules-remove.svg | 75 +++--- docs/images/nf-core-modules-test.svg | 76 +++--- docs/images/nf-core-modules-update.svg | 95 ++++--- docs/images/nf-core-schema-build.svg | 84 +++--- docs/images/nf-core-schema-lint.svg | 78 +++--- docs/images/nf-core-schema-validate.svg | 82 +++--- docs/images/nf-core-sync.svg | 84 +++--- 26 files changed, 1613 insertions(+), 1466 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index 6e8b89d1c2..a956d2c922 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,78 +19,78 @@ font-weight: 700; } - .terminal-1189121654-matrix { + .terminal-440110297-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1189121654-title { + .terminal-440110297-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1189121654-r1 { fill: #c5c8c6 } -.terminal-1189121654-r2 { fill: #98a84b } -.terminal-1189121654-r3 { fill: #9a9b99 } -.terminal-1189121654-r4 { fill: #608ab1 } -.terminal-1189121654-r5 { fill: #d0b344 } -.terminal-1189121654-r6 { fill: #868887 } -.terminal-1189121654-r7 { fill: #cc555a } + .terminal-440110297-r1 { fill: #c5c8c6 } +.terminal-440110297-r2 { fill: #98a84b } +.terminal-440110297-r3 { fill: #9a9b99 } +.terminal-440110297-r4 { fill: #608ab1 } +.terminal-440110297-r5 { fill: #d0b344 } +.terminal-440110297-r6 { fill: #868887 } +.terminal-440110297-r7 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -102,26 +102,26 @@ - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 -INFO     Updated version in 'nextflow.config'bump_version.py:164 - - version         = '1.0dev' - + version = '1.1' - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 +INFO     Updated version in 'nextflow.config'bump_version.py:164 + - version         = '1.0dev' + + version = '1.1' + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 2158fb082e..96278a10cd 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2228597254-matrix { + .terminal-2135273577-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2228597254-title { + .terminal-2135273577-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2228597254-r1 { fill: #c5c8c6 } -.terminal-2228597254-r2 { fill: #98a84b } -.terminal-2228597254-r3 { fill: #9a9b99 } -.terminal-2228597254-r4 { fill: #608ab1 } -.terminal-2228597254-r5 { fill: #d0b344 } -.terminal-2228597254-r6 { fill: #868887 } -.terminal-2228597254-r7 { fill: #98729f } -.terminal-2228597254-r8 { fill: #ff2c7a } -.terminal-2228597254-r9 { fill: #98a84b;font-weight: bold } -.terminal-2228597254-r10 { fill: #1984e9;text-decoration: underline; } + .terminal-2135273577-r1 { fill: #c5c8c6 } +.terminal-2135273577-r2 { fill: #98a84b } +.terminal-2135273577-r3 { fill: #9a9b99 } +.terminal-2135273577-r4 { fill: #608ab1 } +.terminal-2135273577-r5 { fill: #d0b344 } +.terminal-2135273577-r6 { fill: #868887 } +.terminal-2135273577-r7 { fill: #98729f } +.terminal-2135273577-r8 { fill: #ff2c7a } +.terminal-2135273577-r9 { fill: #98a84b;font-weight: bold } +.terminal-2135273577-r10 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,35 +132,35 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 -INFO     Initialising pipeline git repository                                          create.py:538 -INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 -         syncing.                                                                       -INFO    !!!!!! IMPORTANT !!!!!!create.py:227 - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 +INFO     Initialising pipeline git repository                                          create.py:538 +INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 +         syncing.                                                                       +INFO    !!!!!! IMPORTANT !!!!!!create.py:227 + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 5502d4fc46..9fd4ecc75a 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,77 +19,77 @@ font-weight: 700; } - .terminal-2172378614-matrix { + .terminal-3988708441-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2172378614-title { + .terminal-3988708441-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2172378614-r1 { fill: #c5c8c6 } -.terminal-2172378614-r2 { fill: #98a84b } -.terminal-2172378614-r3 { fill: #9a9b99 } -.terminal-2172378614-r4 { fill: #608ab1 } -.terminal-2172378614-r5 { fill: #d0b344 } -.terminal-2172378614-r6 { fill: #868887 } + .terminal-3988708441-r1 { fill: #c5c8c6 } +.terminal-3988708441-r2 { fill: #98a84b } +.terminal-3988708441-r3 { fill: #9a9b99 } +.terminal-3988708441-r4 { fill: #608ab1 } +.terminal-3988708441-r5 { fill: #d0b344 } +.terminal-3988708441-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +101,26 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq'download.py:158 -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                                      download.py:161 -INFO     Downloading centralised configs from GitHub                                 download.py:165 + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq'download.py:158 +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                                      download.py:161 +INFO     Downloading centralised configs from GitHub                                 download.py:165 diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index 96ecbd3426..0815e2d7fe 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,73 +19,73 @@ font-weight: 700; } - .terminal-2229446587-matrix { + .terminal-2111743518-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2229446587-title { + .terminal-2111743518-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2229446587-r1 { fill: #c5c8c6 } -.terminal-2229446587-r2 { fill: #98a84b } -.terminal-2229446587-r3 { fill: #9a9b99 } -.terminal-2229446587-r4 { fill: #608ab1 } -.terminal-2229446587-r5 { fill: #d0b344 } -.terminal-2229446587-r6 { fill: #868887 } -.terminal-2229446587-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2229446587-r8 { fill: #68a0b3;font-weight: bold } + .terminal-2111743518-r1 { fill: #c5c8c6 } +.terminal-2111743518-r2 { fill: #98a84b } +.terminal-2111743518-r3 { fill: #9a9b99 } +.terminal-2111743518-r4 { fill: #608ab1 } +.terminal-2111743518-r5 { fill: #d0b344 } +.terminal-2111743518-r6 { fill: #868887 } +.terminal-2111743518-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2111743518-r8 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,24 +97,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 -         Nextflow config files or profiles                                              - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 +         Nextflow config files or profiles                                              + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 5f84e722e9..df8efdc33f 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-290135093-matrix { + .terminal-1746868888-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-290135093-title { + .terminal-1746868888-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-290135093-r1 { fill: #c5c8c6 } -.terminal-290135093-r2 { fill: #98a84b } -.terminal-290135093-r3 { fill: #9a9b99 } -.terminal-290135093-r4 { fill: #608ab1 } -.terminal-290135093-r5 { fill: #d0b344 } -.terminal-290135093-r6 { fill: #68a0b3;font-weight: bold } -.terminal-290135093-r7 { fill: #868887 } -.terminal-290135093-r8 { fill: #c5c8c6;font-weight: bold } + .terminal-1746868888-r1 { fill: #c5c8c6 } +.terminal-1746868888-r2 { fill: #98a84b } +.terminal-1746868888-r3 { fill: #9a9b99 } +.terminal-1746868888-r4 { fill: #608ab1 } +.terminal-1746868888-r5 { fill: #d0b344 } +.terminal-1746868888-r6 { fill: #68a0b3;font-weight: bold } +.terminal-1746868888-r7 { fill: #868887 } +.terminal-1746868888-r8 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                     licences.py:77 -INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 -         packaged using conda.                                                         -INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                     licences.py:77 +INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 +         packaged using conda.                                                         +INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 1f4b387b45..b7788b02ca 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Testing pipeline: .__init__.py:263 - - -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check - - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc - - -╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/modules…│New version available│ -│fastqc│modules/nf-core/modules…│New version available│ -│multiqc│modules/nf-core/modules…│New version available│ -│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ -│samplesheet_check│modules/local/sampleshe…│when: condition has been │ -│││removed│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 169 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   5 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO     Testing pipeline: .__init__.py:263 + + +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check + + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc + + +╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/modules…│New version available│ +│fastqc│modules/nf-core/modules…│New version available│ +│multiqc│modules/nf-core/modules…│New version available│ +│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ +│samplesheet_check│modules/local/sampleshe…│when: condition has been │ +│││removed│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 169 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   5 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index cbd098b947..def102c509 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-4236301330-matrix { + .terminal-2458374773-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4236301330-title { + .terminal-2458374773-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4236301330-r1 { fill: #c5c8c6 } -.terminal-4236301330-r2 { fill: #98a84b } -.terminal-4236301330-r3 { fill: #9a9b99 } -.terminal-4236301330-r4 { fill: #608ab1 } -.terminal-4236301330-r5 { fill: #d0b344 } -.terminal-4236301330-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4236301330-r7 { fill: #868887 } + .terminal-2458374773-r1 { fill: #c5c8c6 } +.terminal-2458374773-r2 { fill: #98a84b } +.terminal-2458374773-r3 { fill: #9a9b99 } +.terminal-2458374773-r4 { fill: #608ab1 } +.terminal-2458374773-r5 { fill: #d0b344 } +.terminal-2458374773-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2458374773-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ -│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ -│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ -│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    19 │            dev │            - │           - │ -                   │ -│ lncpipe              │    23 │            dev │            - │           - │ -                   │ -│ scflow               │    12 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ +│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    19 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    12 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index da3f02c7db..6b8aa5659b 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-1473353797-matrix { + .terminal-2850002601-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1473353797-title { + .terminal-2850002601-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1473353797-r1 { fill: #c5c8c6 } -.terminal-1473353797-r2 { fill: #98a84b } -.terminal-1473353797-r3 { fill: #9a9b99 } -.terminal-1473353797-r4 { fill: #608ab1 } -.terminal-1473353797-r5 { fill: #d0b344 } -.terminal-1473353797-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1473353797-r7 { fill: #868887 } -.terminal-1473353797-r8 { fill: #868887;font-style: italic; } + .terminal-2850002601-r1 { fill: #c5c8c6 } +.terminal-2850002601-r2 { fill: #98a84b } +.terminal-2850002601-r3 { fill: #9a9b99 } +.terminal-2850002601-r4 { fill: #608ab1 } +.terminal-2850002601-r5 { fill: #d0b344 } +.terminal-2850002601-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2850002601-r7 { fill: #868887 } +.terminal-2850002601-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ -│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ -│ chipseq              │   121 │          1.2.2 │  1 years ago │           - │ -                   │ -│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ chipseq              │   122 │          1.2.2 │  1 years ago │           - │ -                   │ +│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index c5e7e17dd8..943cf41219 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-2711598360-matrix { + .terminal-1258795901-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2711598360-title { + .terminal-1258795901-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2711598360-r1 { fill: #c5c8c6 } -.terminal-2711598360-r2 { fill: #98a84b } -.terminal-2711598360-r3 { fill: #9a9b99 } -.terminal-2711598360-r4 { fill: #608ab1 } -.terminal-2711598360-r5 { fill: #d0b344 } -.terminal-2711598360-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2711598360-r7 { fill: #868887 } -.terminal-2711598360-r8 { fill: #868887;font-style: italic; } + .terminal-1258795901-r1 { fill: #c5c8c6 } +.terminal-1258795901-r2 { fill: #98a84b } +.terminal-1258795901-r3 { fill: #9a9b99 } +.terminal-1258795901-r4 { fill: #608ab1 } +.terminal-1258795901-r5 { fill: #d0b344 } +.terminal-1258795901-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1258795901-r7 { fill: #868887 } +.terminal-1258795901-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ mag                  │    99 │          2.2.1 │   4 days ago │           - │ -                   │ -│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ -│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ -│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ -│ viralrecon           │    74 │            2.5 │ 2 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ mag                  │    99 │          2.2.1 │   5 days ago │           - │ -                   │ +│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ +│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ +│ viralrecon           │    75 │            2.5 │ 2 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index e76e7a0a12..946da37712 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1308324414-matrix { + .terminal-1004368033-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1308324414-title { + .terminal-1004368033-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1308324414-r1 { fill: #c5c8c6 } -.terminal-1308324414-r2 { fill: #98a84b } -.terminal-1308324414-r3 { fill: #9a9b99 } -.terminal-1308324414-r4 { fill: #608ab1 } -.terminal-1308324414-r5 { fill: #d0b344 } -.terminal-1308324414-r6 { fill: #98a84b;font-weight: bold } -.terminal-1308324414-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-1004368033-r1 { fill: #c5c8c6 } +.terminal-1004368033-r2 { fill: #98a84b } +.terminal-1004368033-r3 { fill: #9a9b99 } +.terminal-1004368033-r4 { fill: #608ab1 } +.terminal-1004368033-r5 { fill: #d0b344 } +.terminal-1004368033-r6 { fill: #98a84b;font-weight: bold } +.terminal-1004368033-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 5711bd8431..f8ced5520b 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Looking for test workflow entry points:                             test_yml_builder.py:123 -'tests/modules/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:157 -INFO     Setting env var '$PROFILE' to an empty string as not set.           test_yml_builder.py:301 -         Tests will run with Docker by default. To use Singularity set        -'export PROFILE=singularity' in your shell before running this       -         command.                                                             -INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:325 -nextflow run ./tests/modules/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config  -c  -./tests/modules/fastqc/nextflow.config --outdir /tmp/tmpzotojksy --work-dir /tmp/tmps14qhvf6 + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Looking for test workflow entry points:                             test_yml_builder.py:123 +'tests/modules/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:157 +INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:325 +nextflow run ./tests/modules/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config  -c  +./tests/modules/fastqc/nextflow.config --outdir /tmp/tmpe2lt4618 +-work-dir /tmp/tmpimxa97gt diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 47be3b89c3..741c98771e 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Repository type: modulescreate.py:93 -INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 -responses. ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 -INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 -INFO     Using Singularity container:                                                  create.py:192 -'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Repository type: modulescreate.py:93 +INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 +responses. ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 +INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 +INFO     Using Singularity container:                                                  create.py:192 +'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' +INFO     Created / edited following files:                                             create.py:270 +           ./modules/fastqc/main.nf +           ./modules/fastqc/meta.yml +           ./tests/modules/fastqc/main.nf +           ./tests/modules/fastqc/test.yml +           ./tests/modules/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 9659c49c45..ba471c1863 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - + - - - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Reinstalling modules found in 'modules.json' but missing from           modules_json.py:452 -         directory: 'nf-core/modules/custom/dumpsoftwareversions',                -'nf-core/modules/fastqc', 'nf-core/modules/multiqc' -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 3b8fc97194..d3527d270a 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:483 + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO     Installing 'abacas'install.py:116 +INFO     Include statement: include { ABACAS } from                                   install.py:125 +'../modules/nf-core/modules/abacas/main' diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 24ea8054b9..e16d844a01 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:200 -INFO     Linting module: 'multiqc'__init__.py:204 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ -│                                           ╷                         ╷                            │ -│Module name                              │File path              │Test message              │ -│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ -│multiqc│modules/multiqc/main.nf│Conda update: │ -│││bioconda::multiqc 1.10 -> │ -│││1.13a│ -│                                           ╵                         ╵                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:200 +INFO     Linting module: 'multiqc'__init__.py:204 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ +│                                           ╷                         ╷                            │ +│Module name                              │File path              │Test message              │ +│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ +│multiqc│modules/multiqc/main.nf│Conda update: │ +│││bioconda::multiqc 1.10 -> │ +│││1.13a│ +│                                           ╵                         ╵                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index a8957f7e99..0344143c2b 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'list.py:64 - + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                       list.py:124 + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name         ┃Repository     ┃Version SHA        ┃Message             ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftware… │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ fastqc               │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ multiqc              │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 96b91a83f9..dfbe22e11a 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-277652753-matrix { + .terminal-1837409140-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-277652753-title { + .terminal-1837409140-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-277652753-r1 { fill: #c5c8c6 } -.terminal-277652753-r2 { fill: #98a84b } -.terminal-277652753-r3 { fill: #9a9b99 } -.terminal-277652753-r4 { fill: #608ab1 } -.terminal-277652753-r5 { fill: #d0b344 } -.terminal-277652753-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-277652753-r7 { fill: #868887 } -.terminal-277652753-r8 { fill: #868887;font-style: italic; } + .terminal-1837409140-r1 { fill: #c5c8c6 } +.terminal-1837409140-r2 { fill: #98a84b } +.terminal-1837409140-r3 { fill: #9a9b99 } +.terminal-1837409140-r4 { fill: #608ab1 } +.terminal-1837409140-r5 { fill: #d0b344 } +.terminal-1837409140-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1837409140-r7 { fill: #868887 } +.terminal-1837409140-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):                                list.py:119 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):                                list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index 4e13039d6b..163f41a53d 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -19,70 +19,70 @@ font-weight: 700; } - .terminal-3387661225-matrix { + .terminal-1187224076-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3387661225-title { + .terminal-1187224076-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3387661225-r1 { fill: #c5c8c6 } -.terminal-3387661225-r2 { fill: #98a84b } -.terminal-3387661225-r3 { fill: #9a9b99 } -.terminal-3387661225-r4 { fill: #608ab1 } -.terminal-3387661225-r5 { fill: #d0b344 } -.terminal-3387661225-r6 { fill: #868887 } -.terminal-3387661225-r7 { fill: #00823d;font-weight: bold } -.terminal-3387661225-r8 { fill: #68a0b3;font-weight: bold } + .terminal-1187224076-r1 { fill: #c5c8c6 } +.terminal-1187224076-r2 { fill: #98a84b } +.terminal-1187224076-r3 { fill: #9a9b99 } +.terminal-1187224076-r4 { fill: #608ab1 } +.terminal-1187224076-r5 { fill: #d0b344 } +.terminal-1187224076-r6 { fill: #868887 } +.terminal-1187224076-r7 { fill: #00823d;font-weight: bold } +.terminal-1187224076-r8 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -94,23 +94,23 @@ - + - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                              mulled.py:68 -INFO     Mulled container hash:                                                      __main__.py:828 -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                              mulled.py:68 +INFO     Mulled container hash:                                                      __main__.py:828 +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 757ef4abf9..0c7a4dd9f8 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:571 + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO     Changes in module 'nf-core/modules/fastqc'modules_differ.py:252 +INFO    'modules/nf-core/modules/fastqc/meta.yml' is unchanged                modules_differ.py:257 +INFO     Changes in 'fastqc/main.nf':                                          modules_differ.py:266 + +--- modules/nf-core/modules/fastqc/main.nf ++++ modules/nf-core/modules/fastqc/main.nf +@@ -1,6 +1,6 @@ +process FASTQC {                                                                                   +    tag "$meta.id"                                                                                 +-    label 'process_medium' ++    label 'process_low' + +    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)                                 +    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  + + +INFO     Patch file of 'nf-core/modules/fastqc' written to                              patch.py:115 +'modules/nf-core/modules/fastqc/fastqc.diff' diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 20726d93e6..bc67a96483 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,66 +19,65 @@ font-weight: 700; } - .terminal-1917479430-matrix { + .terminal-1051680509-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1917479430-title { + .terminal-1051680509-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1917479430-r1 { fill: #c5c8c6 } -.terminal-1917479430-r2 { fill: #98a84b } -.terminal-1917479430-r3 { fill: #9a9b99 } -.terminal-1917479430-r4 { fill: #608ab1 } -.terminal-1917479430-r5 { fill: #d0b344 } -.terminal-1917479430-r6 { fill: #292929;font-weight: bold } -.terminal-1917479430-r7 { fill: #868887 } + .terminal-1051680509-r1 { fill: #c5c8c6 } +.terminal-1051680509-r2 { fill: #98a84b } +.terminal-1051680509-r3 { fill: #9a9b99 } +.terminal-1051680509-r4 { fill: #608ab1 } +.terminal-1051680509-r5 { fill: #d0b344 } +.terminal-1051680509-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -90,22 +89,22 @@ - - - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -CRITICAL Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:599 + + + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO     Removing abacas                                                                remove.py:52 diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 20b7f7bb7f..b65ae7a6ad 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-2872056789-matrix { + .terminal-2926910008-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2872056789-title { + .terminal-2926910008-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2872056789-r1 { fill: #c5c8c6 } -.terminal-2872056789-r2 { fill: #98a84b } -.terminal-2872056789-r3 { fill: #9a9b99 } -.terminal-2872056789-r4 { fill: #608ab1 } -.terminal-2872056789-r5 { fill: #d0b344 } -.terminal-2872056789-r6 { fill: #868887 } + .terminal-2926910008-r1 { fill: #c5c8c6 } +.terminal-2926910008-r2 { fill: #98a84b } +.terminal-2926910008-r3 { fill: #9a9b99 } +.terminal-2926910008-r4 { fill: #608ab1 } +.terminal-2926910008-r5 { fill: #d0b344 } +.terminal-2926910008-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,23 +92,23 @@ - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view'module_test.py:184 + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view'module_test.py:184 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 2acd0bdba6..080277cff0 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -ERROR    Could not find a 'main.nf' or 'nextflow.config' file in '.'__main__.py:540 + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + + +INFO    'nf-core/modules/abacas' is already up to date                                update.py:160 +INFO     Updating 'nf-core/modules/custom/dumpsoftwareversions'update.py:516 +INFO     Updating 'nf-core/modules/fastqc'update.py:516 +INFO     Updating 'nf-core/modules/multiqc'update.py:516 +INFO     Updates complete ✨                                                           update.py:242 diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 3c5f4b3370..87f27862ac 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-1219208311-matrix { + .terminal-3538854618-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1219208311-title { + .terminal-3538854618-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1219208311-r1 { fill: #c5c8c6 } -.terminal-1219208311-r2 { fill: #98a84b } -.terminal-1219208311-r3 { fill: #9a9b99 } -.terminal-1219208311-r4 { fill: #608ab1 } -.terminal-1219208311-r5 { fill: #d0b344 } -.terminal-1219208311-r6 { fill: #98a84b;font-weight: bold } -.terminal-1219208311-r7 { fill: #868887 } -.terminal-1219208311-r8 { fill: #868887;font-weight: bold } -.terminal-1219208311-r9 { fill: #4e707b;font-weight: bold } -.terminal-1219208311-r10 { fill: #68a0b3;font-weight: bold } + .terminal-3538854618-r1 { fill: #c5c8c6 } +.terminal-3538854618-r2 { fill: #98a84b } +.terminal-3538854618-r3 { fill: #9a9b99 } +.terminal-3538854618-r4 { fill: #608ab1 } +.terminal-3538854618-r5 { fill: #d0b344 } +.terminal-3538854618-r6 { fill: #98a84b;font-weight: bold } +.terminal-3538854618-r7 { fill: #868887 } +.terminal-3538854618-r8 { fill: #868887;font-weight: bold } +.terminal-3538854618-r9 { fill: #4e707b;font-weight: bold } +.terminal-3538854618-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 27 params)schema.py:95 -INFO     Writing schema with 28 params: './nextflow_schema.json'schema.py:173 + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 27 params)schema.py:95 +INFO     Writing schema with 28 params: './nextflow_schema.json'schema.py:173 diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index aa28d99e0e..0083a44941 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-3729820431-matrix { + .terminal-4207053170-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3729820431-title { + .terminal-4207053170-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3729820431-r1 { fill: #c5c8c6 } -.terminal-3729820431-r2 { fill: #98a84b } -.terminal-3729820431-r3 { fill: #9a9b99 } -.terminal-3729820431-r4 { fill: #608ab1 } -.terminal-3729820431-r5 { fill: #d0b344 } -.terminal-3729820431-r6 { fill: #98a84b;font-weight: bold } -.terminal-3729820431-r7 { fill: #868887 } -.terminal-3729820431-r8 { fill: #868887;font-weight: bold } -.terminal-3729820431-r9 { fill: #4e707b;font-weight: bold } + .terminal-4207053170-r1 { fill: #c5c8c6 } +.terminal-4207053170-r2 { fill: #98a84b } +.terminal-4207053170-r3 { fill: #9a9b99 } +.terminal-4207053170-r4 { fill: #608ab1 } +.terminal-4207053170-r5 { fill: #d0b344 } +.terminal-4207053170-r6 { fill: #98a84b;font-weight: bold } +.terminal-4207053170-r7 { fill: #868887 } +.terminal-4207053170-r8 { fill: #868887;font-weight: bold } +.terminal-4207053170-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 41376f5bdb..2e089460a0 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-3209931885-matrix { + .terminal-229550800-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3209931885-title { + .terminal-229550800-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3209931885-r1 { fill: #c5c8c6 } -.terminal-3209931885-r2 { fill: #98a84b } -.terminal-3209931885-r3 { fill: #9a9b99 } -.terminal-3209931885-r4 { fill: #608ab1 } -.terminal-3209931885-r5 { fill: #d0b344 } -.terminal-3209931885-r6 { fill: #98a84b;font-weight: bold } -.terminal-3209931885-r7 { fill: #868887 } -.terminal-3209931885-r8 { fill: #868887;font-weight: bold } -.terminal-3209931885-r9 { fill: #4e707b;font-weight: bold } + .terminal-229550800-r1 { fill: #c5c8c6 } +.terminal-229550800-r2 { fill: #98a84b } +.terminal-229550800-r3 { fill: #9a9b99 } +.terminal-229550800-r4 { fill: #608ab1 } +.terminal-229550800-r5 { fill: #d0b344 } +.terminal-229550800-r6 { fill: #98a84b;font-weight: bold } +.terminal-229550800-r7 { fill: #868887 } +.terminal-229550800-r8 { fill: #868887;font-weight: bold } +.terminal-229550800-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 -INFO    [✓] Input parameters look validschema.py:213 + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 +INFO    [✓] Input parameters look validschema.py:213 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 1b6f3a9e83..ec1ac051e8 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,73 +19,73 @@ font-weight: 700; } - .terminal-2822682943-matrix { + .terminal-3145054114-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2822682943-title { + .terminal-3145054114-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2822682943-r1 { fill: #c5c8c6 } -.terminal-2822682943-r2 { fill: #98a84b } -.terminal-2822682943-r3 { fill: #9a9b99 } -.terminal-2822682943-r4 { fill: #608ab1 } -.terminal-2822682943-r5 { fill: #d0b344 } -.terminal-2822682943-r6 { fill: #98729f } -.terminal-2822682943-r7 { fill: #ff2c7a } -.terminal-2822682943-r8 { fill: #868887 } + .terminal-3145054114-r1 { fill: #c5c8c6 } +.terminal-3145054114-r2 { fill: #98a84b } +.terminal-3145054114-r3 { fill: #9a9b99 } +.terminal-3145054114-r4 { fill: #608ab1 } +.terminal-3145054114-r5 { fill: #d0b344 } +.terminal-3145054114-r6 { fill: #98729f } +.terminal-3145054114-r7 { fill: #ff2c7a } +.terminal-3145054114-r8 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,24 +97,24 @@ - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 -INFO     Making a new template pipeline using pipeline variables                         sync.py:223 + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 +INFO     Making a new template pipeline using pipeline variables                         sync.py:223 From bf5bdb57bf1e0f338b8098ee27423e07d4179e3a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Aug 2022 15:38:10 +0200 Subject: [PATCH 041/854] Bump to v2.6dev --- .github/RELEASE_CHECKLIST.md | 9 +++++---- CHANGELOG.md | 10 ++++++++++ setup.py | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index e086d286aa..c5b3464b83 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -2,16 +2,17 @@ 1. Check issue milestones to see outstanding issues to resolve if possible or transfer to the milestones for the next release e.g. [`v1.9`](https://github.com/nf-core/tools/issues?q=is%3Aopen+is%3Aissue+milestone%3A1.9) 2. Most importantly, pick an undeniably outstanding [name](http://www.codenamegenerator.com/) for the release where _Prefix_ = _Metal_ and _Dictionary_ = _Animal_. -3. Check whether the GitHub Actions workflow scripts need updating of the Nextflow versions +3. Check the [pipeline health page](https://nf-co.re/pipeline_health) to make sure that all repos look sane (missing `TEMPLATE` branches etc) 4. Create a PR to `dev` to bump the version in `CHANGELOG.md` and `setup.py`. 5. Make sure all CI tests are passing! 6. Create a PR from `dev` to `master` 7. Make sure all CI tests are passing again (additional tests are run on PRs to `master`) 8. Request review (2 approvals required) 9. Merge the PR into `master` -10. Wait for CI tests on the commit to passed -11. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR. -12. Create a new release copying the `CHANGELOG` for that release into the description section. +10. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button) +11. Wait for CI tests on the commit to passed +12. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR. +13. Create a new release copying the `CHANGELOG` for that release into the description section. ## After release diff --git a/CHANGELOG.md b/CHANGELOG.md index aa177f6ecd..8132f37c05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # nf-core/tools: Changelog +## v2.6dev + +### Template + +### Linting + +### General + +### Modules + ## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] ### Template diff --git a/setup.py b/setup.py index ac95a82c90..0759633a0b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.5" +version = "2.6dev" with open("README.md") as f: readme = f.read() From 9c942b81f96cbe32f64e38b9c6be96e0e90e5865 Mon Sep 17 00:00:00 2001 From: "Maxime U. Garcia" Date: Tue, 30 Aug 2022 16:55:48 +0200 Subject: [PATCH 042/854] Update CITATION.cff --- CITATION.cff | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 4533e2f28c..017666c018 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -13,8 +13,8 @@ authors: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen @@ -39,8 +39,8 @@ prefered-citation: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen From 23bd7e4306822cfec7a6c5e1b2a80725a5351d1c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 30 Aug 2022 17:39:24 +0200 Subject: [PATCH 043/854] add pyproject.toml file for black custom parameters --- nf_core/pipeline-template/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 nf_core/pipeline-template/pyproject.toml diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml new file mode 100644 index 0000000000..53cc9af325 --- /dev/null +++ b/nf_core/pipeline-template/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 120 +target_version = ["py37", "py38", "py39", "py310"] From 732bbd0fdd475dab0f803342ddecf991daf0b30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 31 Aug 2022 09:34:28 +0200 Subject: [PATCH 044/854] Apply suggestion from code review Co-authored-by: Phil Ewels --- nf_core/pipeline-template/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 53cc9af325..1ed87cee5b 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -1,3 +1,5 @@ +# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. [tool.black] line-length = 120 target_version = ["py37", "py38", "py39", "py310"] From 2983c47a9f2b7764f498caae44dbb323fd90021c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 31 Aug 2022 10:55:06 +0200 Subject: [PATCH 045/854] bump to 2.5.1 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8132f37c05..eb7e437216 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ ### Modules +## [v2.5 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-31] + +- Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)). + ## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] ### Template diff --git a/setup.py b/setup.py index 0759633a0b..f3335db354 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.6dev" +version = "2.5.1" with open("README.md") as f: readme = f.read() From 20d9b73a3cd3a760e32ac839fa79b646de362f8e Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 30 Aug 2022 15:51:12 +0200 Subject: [PATCH 046/854] use GitHub PyPI publish action release v1 --- .github/workflows/deploy-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 391a8ef94f..8d951b416d 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -31,6 +31,6 @@ jobs: - name: Publish dist to PyPI if: github.repository == 'nf-core/tools' - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password }} From 2d51c96f89114e1a499a4b7591cf4bd3507dd4be Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 30 Aug 2022 15:58:22 +0200 Subject: [PATCH 047/854] update GitHub PyPI release action to v1 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8132f37c05..8948fa0515 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### General +- Update GitHub PyPI package release action to v1 ([#1785](https://github.com/nf-core/tools/pull/1785)) + ### Modules ## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] From 6fabd3aecced72a8c57248470d848154b62cb13e Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 31 Aug 2022 11:28:22 +0200 Subject: [PATCH 048/854] use nf-core tools isort settings in pipeline template --- nf_core/pipeline-template/pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 1ed87cee5b..0d62beb6f9 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -3,3 +3,8 @@ [tool.black] line-length = 120 target_version = ["py37", "py38", "py39", "py310"] + +[tool.isort] +profile = "black" +known_first_party = ["nf_core"] +multi_line_output = 3 From d79c14e43fd72cf786cfdfbecce760ecf9c58e80 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 31 Aug 2022 12:15:26 +0200 Subject: [PATCH 049/854] update changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66dc35aa0c..e3aa9da72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,13 @@ ### General -- Update GitHub PyPI package release action to v1 ([#1785](https://github.com/nf-core/tools/pull/1785)) - ### Modules ## [v2.5 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-31] -- Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)). +- Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)) +- Add isort options to pyproject.toml ([#1792](https://github.com/nf-core/tools/pull/1792)) +- Update GitHub PyPI package release action to v1 ([#1785](https://github.com/nf-core/tools/pull/1785)) ## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] From c40a0c93c3987e8c9e5baba622a50900ffd41bee Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 31 Aug 2022 12:45:02 +0200 Subject: [PATCH 050/854] remove v2.6dev from changelog --- CHANGELOG.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3aa9da72f..a0232b58c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,5 @@ # nf-core/tools: Changelog -## v2.6dev - -### Template - -### Linting - -### General - -### Modules - ## [v2.5 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-31] - Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)) From 8fed2ae0fe60e40f07177dc8872b149371c653af Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 31 Aug 2022 13:21:23 +0200 Subject: [PATCH 051/854] lint pyproject.toml file exists and content --- nf_core/lint/files_exist.py | 2 ++ nf_core/lint/files_unchanged.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index a65f28f11d..bc0f732b2e 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -73,6 +73,7 @@ def files_exist(self): .github/workflows/awstest.yml .github/workflows/awsfulltest.yml lib/WorkflowPIPELINE.groovy + pyproject.toml Files that *must not* be present: @@ -174,6 +175,7 @@ def files_exist(self): [os.path.join(".github", "workflows", "awsfulltest.yml")], [os.path.join("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")], ["modules.json"], + ["pyproject.toml"], ] # List of strings. Fails / warns if any of the strings exist. diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 7c82d9961b..8b63a5769e 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -50,6 +50,7 @@ def files_unchanged(self): .gitignore .prettierignore + pyproject.toml .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting the ``files_unchanged`` key as follows in your ``.nf-core.yml`` config file. For example: @@ -110,7 +111,7 @@ def files_unchanged(self): [os.path.join("lib", "NfcoreTemplate.groovy")], ] files_partial = [ - [".gitignore", ".prettierignore"], + [".gitignore", ".prettierignore", "pyproject.toml"], ] # Only show error messages from pipeline creation From d5cbde0f8ef80bb9e78c44fba3f074e9d7f668eb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 31 Aug 2022 13:25:17 +0200 Subject: [PATCH 052/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3aa9da72f..ccfac9cd04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)) - Add isort options to pyproject.toml ([#1792](https://github.com/nf-core/tools/pull/1792)) +- Lint pyproject.toml file exists and content ([#1795](https://github.com/nf-core/tools/pull/1795)) - Update GitHub PyPI package release action to v1 ([#1785](https://github.com/nf-core/tools/pull/1785)) ## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] From 2d7360fc9fa348ca7ba5c2335e5b5872d9a20029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 31 Aug 2022 14:15:37 +0200 Subject: [PATCH 053/854] Update CHANGELOG.md Co-authored-by: Phil Ewels --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0232b58c8..11cdbf0683 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## [v2.5 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-31] +## [v2.5.1 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5.1) - [2022-08-31] - Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)) - Add isort options to pyproject.toml ([#1792](https://github.com/nf-core/tools/pull/1792)) From 8d9bae6d8dd0a78b9a40e2f6cfe40ddb16dbfa2a Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 30 Aug 2022 17:39:37 +0200 Subject: [PATCH 054/854] be compliant with black defaults --- nf_core/pipeline-template/bin/check_samplesheet.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py index 9a8b896239..6e778747c0 100755 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ b/nf_core/pipeline-template/bin/check_samplesheet.py @@ -98,7 +98,9 @@ def _validate_pair(self, row): """Assert that read pairs have the same file extension. Report pair status.""" if row[self._first_col] and row[self._second_col]: row[self._single_col] = False - if Path(row[self._first_col]).suffixes[-2:] != Path(row[self._second_col]).suffixes[-2:]: + first_col_suffix = Path(row[self._first_col]).suffixes[-2:] + second_col_suffix = Path(row[self._second_col]).suffixes[-2:] + if first_col_suffix != second_col_suffix: raise AssertionError("FASTQ pairs must have the same file extensions.") else: row[self._single_col] = True @@ -195,7 +197,8 @@ def check_samplesheet(file_in, file_out): reader = csv.DictReader(in_handle, dialect=sniff_format(in_handle)) # Validate the existence of the expected header columns. if not required_columns.issubset(reader.fieldnames): - logger.critical(f"The sample sheet **must** contain the column headers: {', '.join(required_columns)}.") + req_cols = ", ".join(required_columns) + logger.critical(f"The sample sheet **must** contain these column headers: {req_cols}.") sys.exit(1) # Validate each row. checker = RowChecker() From 9d30b2b26423ffffd4355fbf3842faf8b59a1774 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 30 Aug 2022 18:00:03 +0200 Subject: [PATCH 055/854] remove unnecessary f-string --- nf_core/pipeline-template/bin/check_samplesheet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py index 6e778747c0..11b155723a 100755 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ b/nf_core/pipeline-template/bin/check_samplesheet.py @@ -159,7 +159,7 @@ def sniff_format(handle): handle.seek(0) sniffer = csv.Sniffer() if not sniffer.has_header(peek): - logger.critical(f"The given sample sheet does not appear to contain a header.") + logger.critical("The given sample sheet does not appear to contain a header.") sys.exit(1) dialect = sniffer.sniff(peek) return dialect From bc56cdd35077e408a0a7c90ec07fcdff7c2d1f3b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 Sep 2022 11:16:43 +0000 Subject: [PATCH 056/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 90 +++--- docs/images/nf-core-create.svg | 132 ++++----- docs/images/nf-core-download.svg | 88 +++--- docs/images/nf-core-launch-rnaseq.svg | 84 +++--- docs/images/nf-core-licences.svg | 132 ++++----- docs/images/nf-core-lint.svg | 278 ++++++++++--------- docs/images/nf-core-list-rna.svg | 118 ++++---- docs/images/nf-core-list-stars.svg | 104 +++---- docs/images/nf-core-list.svg | 108 +++---- docs/images/nf-core-modules-bump-version.svg | 106 +++---- docs/images/nf-core-modules-create-test.svg | 104 +++---- docs/images/nf-core-modules-create.svg | 126 ++++----- docs/images/nf-core-modules-info.svg | 206 +++++++------- docs/images/nf-core-modules-install.svg | 82 +++--- docs/images/nf-core-modules-lint.svg | 192 ++++++------- docs/images/nf-core-modules-list-local.svg | 132 ++++----- docs/images/nf-core-modules-list-remote.svg | 132 ++++----- docs/images/nf-core-modules-mulled.svg | 80 +++--- docs/images/nf-core-modules-patch.svg | 146 +++++----- docs/images/nf-core-modules-remove.svg | 72 ++--- docs/images/nf-core-modules-test.svg | 76 ++--- docs/images/nf-core-modules-update.svg | 88 +++--- docs/images/nf-core-schema-build.svg | 84 +++--- docs/images/nf-core-schema-lint.svg | 78 +++--- docs/images/nf-core-schema-validate.svg | 82 +++--- docs/images/nf-core-sync.svg | 84 +++--- 26 files changed, 1506 insertions(+), 1498 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index a956d2c922..6ccdc52346 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,78 +19,78 @@ font-weight: 700; } - .terminal-440110297-matrix { + .terminal-218664248-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-440110297-title { + .terminal-218664248-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-440110297-r1 { fill: #c5c8c6 } -.terminal-440110297-r2 { fill: #98a84b } -.terminal-440110297-r3 { fill: #9a9b99 } -.terminal-440110297-r4 { fill: #608ab1 } -.terminal-440110297-r5 { fill: #d0b344 } -.terminal-440110297-r6 { fill: #868887 } -.terminal-440110297-r7 { fill: #cc555a } + .terminal-218664248-r1 { fill: #c5c8c6 } +.terminal-218664248-r2 { fill: #98a84b } +.terminal-218664248-r3 { fill: #9a9b99 } +.terminal-218664248-r4 { fill: #608ab1 } +.terminal-218664248-r5 { fill: #d0b344 } +.terminal-218664248-r6 { fill: #868887 } +.terminal-218664248-r7 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -102,26 +102,26 @@ - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 -INFO     Updated version in 'nextflow.config'bump_version.py:164 - - version         = '1.0dev' - + version = '1.1' - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 +INFO     Updated version in 'nextflow.config'bump_version.py:164 + - version         = '1.0dev' + + version = '1.1' + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 96278a10cd..7788ad2acf 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2135273577-matrix { + .terminal-1555017928-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2135273577-title { + .terminal-1555017928-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2135273577-r1 { fill: #c5c8c6 } -.terminal-2135273577-r2 { fill: #98a84b } -.terminal-2135273577-r3 { fill: #9a9b99 } -.terminal-2135273577-r4 { fill: #608ab1 } -.terminal-2135273577-r5 { fill: #d0b344 } -.terminal-2135273577-r6 { fill: #868887 } -.terminal-2135273577-r7 { fill: #98729f } -.terminal-2135273577-r8 { fill: #ff2c7a } -.terminal-2135273577-r9 { fill: #98a84b;font-weight: bold } -.terminal-2135273577-r10 { fill: #1984e9;text-decoration: underline; } + .terminal-1555017928-r1 { fill: #c5c8c6 } +.terminal-1555017928-r2 { fill: #98a84b } +.terminal-1555017928-r3 { fill: #9a9b99 } +.terminal-1555017928-r4 { fill: #608ab1 } +.terminal-1555017928-r5 { fill: #d0b344 } +.terminal-1555017928-r6 { fill: #868887 } +.terminal-1555017928-r7 { fill: #98729f } +.terminal-1555017928-r8 { fill: #ff2c7a } +.terminal-1555017928-r9 { fill: #98a84b;font-weight: bold } +.terminal-1555017928-r10 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,35 +132,35 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 -INFO     Initialising pipeline git repository                                          create.py:538 -INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 -         syncing.                                                                       -INFO    !!!!!! IMPORTANT !!!!!!create.py:227 - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 +INFO     Initialising pipeline git repository                                          create.py:538 +INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 +         syncing.                                                                       +INFO    !!!!!! IMPORTANT !!!!!!create.py:227 + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 9fd4ecc75a..d72b0ac0d6 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,77 +19,77 @@ font-weight: 700; } - .terminal-3988708441-matrix { + .terminal-2067913912-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3988708441-title { + .terminal-2067913912-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3988708441-r1 { fill: #c5c8c6 } -.terminal-3988708441-r2 { fill: #98a84b } -.terminal-3988708441-r3 { fill: #9a9b99 } -.terminal-3988708441-r4 { fill: #608ab1 } -.terminal-3988708441-r5 { fill: #d0b344 } -.terminal-3988708441-r6 { fill: #868887 } + .terminal-2067913912-r1 { fill: #c5c8c6 } +.terminal-2067913912-r2 { fill: #98a84b } +.terminal-2067913912-r3 { fill: #9a9b99 } +.terminal-2067913912-r4 { fill: #608ab1 } +.terminal-2067913912-r5 { fill: #d0b344 } +.terminal-2067913912-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +101,26 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq'download.py:158 -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                                      download.py:161 -INFO     Downloading centralised configs from GitHub                                 download.py:165 + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq'download.py:158 +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                                      download.py:161 +INFO     Downloading centralised configs from GitHub                                 download.py:165 diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index 0815e2d7fe..06008efdc8 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,73 +19,73 @@ font-weight: 700; } - .terminal-2111743518-matrix { + .terminal-2752685693-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2111743518-title { + .terminal-2752685693-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2111743518-r1 { fill: #c5c8c6 } -.terminal-2111743518-r2 { fill: #98a84b } -.terminal-2111743518-r3 { fill: #9a9b99 } -.terminal-2111743518-r4 { fill: #608ab1 } -.terminal-2111743518-r5 { fill: #d0b344 } -.terminal-2111743518-r6 { fill: #868887 } -.terminal-2111743518-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2111743518-r8 { fill: #68a0b3;font-weight: bold } + .terminal-2752685693-r1 { fill: #c5c8c6 } +.terminal-2752685693-r2 { fill: #98a84b } +.terminal-2752685693-r3 { fill: #9a9b99 } +.terminal-2752685693-r4 { fill: #608ab1 } +.terminal-2752685693-r5 { fill: #d0b344 } +.terminal-2752685693-r6 { fill: #868887 } +.terminal-2752685693-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2752685693-r8 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,24 +97,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 -         Nextflow config files or profiles                                              - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 +         Nextflow config files or profiles                                              + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index df8efdc33f..ab893e956e 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-1746868888-matrix { + .terminal-2684361463-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1746868888-title { + .terminal-2684361463-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1746868888-r1 { fill: #c5c8c6 } -.terminal-1746868888-r2 { fill: #98a84b } -.terminal-1746868888-r3 { fill: #9a9b99 } -.terminal-1746868888-r4 { fill: #608ab1 } -.terminal-1746868888-r5 { fill: #d0b344 } -.terminal-1746868888-r6 { fill: #68a0b3;font-weight: bold } -.terminal-1746868888-r7 { fill: #868887 } -.terminal-1746868888-r8 { fill: #c5c8c6;font-weight: bold } + .terminal-2684361463-r1 { fill: #c5c8c6 } +.terminal-2684361463-r2 { fill: #98a84b } +.terminal-2684361463-r3 { fill: #9a9b99 } +.terminal-2684361463-r4 { fill: #608ab1 } +.terminal-2684361463-r5 { fill: #d0b344 } +.terminal-2684361463-r6 { fill: #68a0b3;font-weight: bold } +.terminal-2684361463-r7 { fill: #868887 } +.terminal-2684361463-r8 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                     licences.py:77 -INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 -         packaged using conda.                                                         -INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                     licences.py:77 +INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 +         packaged using conda.                                                         +INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index b7788b02ca..ff0b82569c 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Testing pipeline: .__init__.py:263 - - -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check - - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc - - -╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/modules…│New version available│ -│fastqc│modules/nf-core/modules…│New version available│ -│multiqc│modules/nf-core/modules…│New version available│ -│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ -│samplesheet_check│modules/local/sampleshe…│when: condition has been │ -│││removed│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 169 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   5 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO     Testing pipeline: .__init__.py:263 + + +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check + + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc + + +╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/modules…│New version available│ +│fastqc│modules/nf-core/modules…│New version available│ +│multiqc│modules/nf-core/modules…│New version available│ +│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ +│samplesheet_check│modules/local/sampleshe…│when: condition has been │ +│││removed│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 171 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   5 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index def102c509..5060abb2f9 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-2458374773-matrix { + .terminal-461755092-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2458374773-title { + .terminal-461755092-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2458374773-r1 { fill: #c5c8c6 } -.terminal-2458374773-r2 { fill: #98a84b } -.terminal-2458374773-r3 { fill: #9a9b99 } -.terminal-2458374773-r4 { fill: #608ab1 } -.terminal-2458374773-r5 { fill: #d0b344 } -.terminal-2458374773-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2458374773-r7 { fill: #868887 } + .terminal-461755092-r1 { fill: #c5c8c6 } +.terminal-461755092-r2 { fill: #98a84b } +.terminal-461755092-r3 { fill: #9a9b99 } +.terminal-461755092-r4 { fill: #608ab1 } +.terminal-461755092-r5 { fill: #d0b344 } +.terminal-461755092-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-461755092-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ -│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ -│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ -│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    19 │            dev │            - │           - │ -                   │ -│ lncpipe              │    23 │            dev │            - │           - │ -                   │ -│ scflow               │    12 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ +│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    19 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    12 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 6b8aa5659b..c0a53819d9 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-2850002601-matrix { + .terminal-478910217-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2850002601-title { + .terminal-478910217-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2850002601-r1 { fill: #c5c8c6 } -.terminal-2850002601-r2 { fill: #98a84b } -.terminal-2850002601-r3 { fill: #9a9b99 } -.terminal-2850002601-r4 { fill: #608ab1 } -.terminal-2850002601-r5 { fill: #d0b344 } -.terminal-2850002601-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2850002601-r7 { fill: #868887 } -.terminal-2850002601-r8 { fill: #868887;font-style: italic; } + .terminal-478910217-r1 { fill: #c5c8c6 } +.terminal-478910217-r2 { fill: #98a84b } +.terminal-478910217-r3 { fill: #9a9b99 } +.terminal-478910217-r4 { fill: #608ab1 } +.terminal-478910217-r5 { fill: #d0b344 } +.terminal-478910217-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-478910217-r7 { fill: #868887 } +.terminal-478910217-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ -│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ -│ chipseq              │   122 │          1.2.2 │  1 years ago │           - │ -                   │ -│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ +│ sarek                │   187 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ chipseq              │   122 │          1.2.2 │  1 years ago │           - │ -                   │ +│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 943cf41219..bf3e9b27a4 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-1258795901-matrix { + .terminal-3459298259-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1258795901-title { + .terminal-3459298259-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1258795901-r1 { fill: #c5c8c6 } -.terminal-1258795901-r2 { fill: #98a84b } -.terminal-1258795901-r3 { fill: #9a9b99 } -.terminal-1258795901-r4 { fill: #608ab1 } -.terminal-1258795901-r5 { fill: #d0b344 } -.terminal-1258795901-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1258795901-r7 { fill: #868887 } -.terminal-1258795901-r8 { fill: #868887;font-style: italic; } + .terminal-3459298259-r1 { fill: #c5c8c6 } +.terminal-3459298259-r2 { fill: #98a84b } +.terminal-3459298259-r3 { fill: #9a9b99 } +.terminal-3459298259-r4 { fill: #608ab1 } +.terminal-3459298259-r5 { fill: #d0b344 } +.terminal-3459298259-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3459298259-r7 { fill: #868887 } +.terminal-3459298259-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ mag                  │    99 │          2.2.1 │   5 days ago │           - │ -                   │ -│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ -│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ -│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ -│ viralrecon           │    75 │            2.5 │ 2 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ mag                  │   100 │          2.2.1 │   1 week ago │           - │ -                   │ +│ sarek                │   187 │          3.0.1 │  2 weeks ago │           - │ -                   │ +│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ +│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ +│ viralrecon           │    75 │            2.5 │ 2 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 946da37712..eb4a1823c2 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1004368033-matrix { + .terminal-2048094464-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1004368033-title { + .terminal-2048094464-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1004368033-r1 { fill: #c5c8c6 } -.terminal-1004368033-r2 { fill: #98a84b } -.terminal-1004368033-r3 { fill: #9a9b99 } -.terminal-1004368033-r4 { fill: #608ab1 } -.terminal-1004368033-r5 { fill: #d0b344 } -.terminal-1004368033-r6 { fill: #98a84b;font-weight: bold } -.terminal-1004368033-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-2048094464-r1 { fill: #c5c8c6 } +.terminal-2048094464-r2 { fill: #98a84b } +.terminal-2048094464-r3 { fill: #9a9b99 } +.terminal-2048094464-r4 { fill: #608ab1 } +.terminal-2048094464-r5 { fill: #d0b344 } +.terminal-2048094464-r6 { fill: #98a84b;font-weight: bold } +.terminal-2048094464-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index f8ced5520b..0ed9272626 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-1860821773-matrix { + .terminal-3996574694-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1860821773-title { + .terminal-3996574694-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1860821773-r1 { fill: #c5c8c6 } -.terminal-1860821773-r2 { fill: #98a84b } -.terminal-1860821773-r3 { fill: #9a9b99 } -.terminal-1860821773-r4 { fill: #608ab1 } -.terminal-1860821773-r5 { fill: #d0b344 } -.terminal-1860821773-r6 { fill: #868887 } -.terminal-1860821773-r7 { fill: #ff2c7a } -.terminal-1860821773-r8 { fill: #98729f } + .terminal-3996574694-r1 { fill: #c5c8c6 } +.terminal-3996574694-r2 { fill: #98a84b } +.terminal-3996574694-r3 { fill: #9a9b99 } +.terminal-3996574694-r4 { fill: #608ab1 } +.terminal-3996574694-r5 { fill: #d0b344 } +.terminal-3996574694-r6 { fill: #868887 } +.terminal-3996574694-r7 { fill: #ff2c7a } +.terminal-3996574694-r8 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Looking for test workflow entry points:                             test_yml_builder.py:123 -'tests/modules/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:157 -INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:325 -nextflow run ./tests/modules/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config  -c  -./tests/modules/fastqc/nextflow.config --outdir /tmp/tmpe2lt4618 --work-dir /tmp/tmpimxa97gt + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Looking for test workflow entry points:                             test_yml_builder.py:123 +'tests/modules/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:157 +INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:325 +nextflow run ./tests/modules/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config  -c  +./tests/modules/fastqc/nextflow.config --outdir /tmp/tmpfzia90r4 +-work-dir /tmp/tmpq2f6vtpl diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 741c98771e..55dfaeff70 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -19,104 +19,104 @@ font-weight: 700; } - .terminal-430419789-matrix { + .terminal-3763908524-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-430419789-title { + .terminal-3763908524-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-430419789-r1 { fill: #c5c8c6 } -.terminal-430419789-r2 { fill: #98a84b } -.terminal-430419789-r3 { fill: #9a9b99 } -.terminal-430419789-r4 { fill: #608ab1 } -.terminal-430419789-r5 { fill: #d0b344 } -.terminal-430419789-r6 { fill: #868887 } -.terminal-430419789-r7 { fill: #68a0b3;font-weight: bold } -.terminal-430419789-r8 { fill: #98729f } -.terminal-430419789-r9 { fill: #ff2c7a } + .terminal-3763908524-r1 { fill: #c5c8c6 } +.terminal-3763908524-r2 { fill: #98a84b } +.terminal-3763908524-r3 { fill: #9a9b99 } +.terminal-3763908524-r4 { fill: #608ab1 } +.terminal-3763908524-r5 { fill: #d0b344 } +.terminal-3763908524-r6 { fill: #868887 } +.terminal-3763908524-r7 { fill: #68a0b3;font-weight: bold } +.terminal-3763908524-r8 { fill: #98729f } +.terminal-3763908524-r9 { fill: #ff2c7a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -128,34 +128,34 @@ - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Repository type: modulescreate.py:93 -INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 -responses. ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 -INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 -INFO     Using Singularity container:                                                  create.py:192 -'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' -INFO     Created / edited following files:                                             create.py:270 -           ./modules/fastqc/main.nf -           ./modules/fastqc/meta.yml -           ./tests/modules/fastqc/main.nf -           ./tests/modules/fastqc/test.yml -           ./tests/modules/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Repository type: modulescreate.py:93 +INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 +responses. ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 +INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 +INFO     Using Singularity container:                                                  create.py:192 +'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' +INFO     Created / edited following files:                                             create.py:270 +           ./modules/fastqc/main.nf +           ./modules/fastqc/meta.yml +           ./tests/modules/fastqc/main.nf +           ./tests/modules/fastqc/test.yml +           ./tests/modules/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index ba471c1863..96b9648e87 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-1198297747-matrix { + .terminal-2531955442-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1198297747-title { + .terminal-2531955442-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1198297747-r1 { fill: #c5c8c6 } -.terminal-1198297747-r2 { fill: #98a84b } -.terminal-1198297747-r3 { fill: #9a9b99 } -.terminal-1198297747-r4 { fill: #608ab1 } -.terminal-1198297747-r5 { fill: #d0b344 } -.terminal-1198297747-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1198297747-r7 { fill: #98a84b;font-weight: bold } -.terminal-1198297747-r8 { fill: #868887 } -.terminal-1198297747-r9 { fill: #d08442 } -.terminal-1198297747-r10 { fill: #868887;font-style: italic; } -.terminal-1198297747-r11 { fill: #98729f } + .terminal-2531955442-r1 { fill: #c5c8c6 } +.terminal-2531955442-r2 { fill: #98a84b } +.terminal-2531955442-r3 { fill: #9a9b99 } +.terminal-2531955442-r4 { fill: #608ab1 } +.terminal-2531955442-r5 { fill: #d0b344 } +.terminal-2531955442-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2531955442-r7 { fill: #98a84b;font-weight: bold } +.terminal-2531955442-r8 { fill: #868887 } +.terminal-2531955442-r9 { fill: #d08442 } +.terminal-2531955442-r10 { fill: #868887;font-style: italic; } +.terminal-2531955442-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index d3527d270a..8337d42ea4 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-366252011-matrix { + .terminal-436899914-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-366252011-title { + .terminal-436899914-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-366252011-r1 { fill: #c5c8c6 } -.terminal-366252011-r2 { fill: #98a84b } -.terminal-366252011-r3 { fill: #9a9b99 } -.terminal-366252011-r4 { fill: #608ab1 } -.terminal-366252011-r5 { fill: #d0b344 } -.terminal-366252011-r6 { fill: #868887 } -.terminal-366252011-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-436899914-r1 { fill: #c5c8c6 } +.terminal-436899914-r2 { fill: #98a84b } +.terminal-436899914-r3 { fill: #9a9b99 } +.terminal-436899914-r4 { fill: #608ab1 } +.terminal-436899914-r5 { fill: #d0b344 } +.terminal-436899914-r6 { fill: #868887 } +.terminal-436899914-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Installing 'abacas'install.py:116 -INFO     Include statement: include { ABACAS } from                                   install.py:125 -'../modules/nf-core/modules/abacas/main' + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO     Installing 'abacas'install.py:116 +INFO     Include statement: include { ABACAS } from                                   install.py:125 +'../modules/nf-core/modules/abacas/main' diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index e16d844a01..c20e8acb95 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -19,151 +19,151 @@ font-weight: 700; } - .terminal-3614898378-matrix { + .terminal-1271396649-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3614898378-title { + .terminal-1271396649-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3614898378-r1 { fill: #c5c8c6 } -.terminal-3614898378-r2 { fill: #98a84b } -.terminal-3614898378-r3 { fill: #9a9b99 } -.terminal-3614898378-r4 { fill: #608ab1 } -.terminal-3614898378-r5 { fill: #d0b344 } -.terminal-3614898378-r6 { fill: #868887 } -.terminal-3614898378-r7 { fill: #608ab1;font-weight: bold } -.terminal-3614898378-r8 { fill: #98729f } -.terminal-3614898378-r9 { fill: #d0b344;font-weight: bold } -.terminal-3614898378-r10 { fill: #8d7b39 } -.terminal-3614898378-r11 { fill: #fdfdc5 } -.terminal-3614898378-r12 { fill: #c5c8c6;font-weight: bold } -.terminal-3614898378-r13 { fill: #98a84b;font-weight: bold } -.terminal-3614898378-r14 { fill: #cc555a } + .terminal-1271396649-r1 { fill: #c5c8c6 } +.terminal-1271396649-r2 { fill: #98a84b } +.terminal-1271396649-r3 { fill: #9a9b99 } +.terminal-1271396649-r4 { fill: #608ab1 } +.terminal-1271396649-r5 { fill: #d0b344 } +.terminal-1271396649-r6 { fill: #868887 } +.terminal-1271396649-r7 { fill: #608ab1;font-weight: bold } +.terminal-1271396649-r8 { fill: #98729f } +.terminal-1271396649-r9 { fill: #d0b344;font-weight: bold } +.terminal-1271396649-r10 { fill: #8d7b39 } +.terminal-1271396649-r11 { fill: #fdfdc5 } +.terminal-1271396649-r12 { fill: #c5c8c6;font-weight: bold } +.terminal-1271396649-r13 { fill: #98a84b;font-weight: bold } +.terminal-1271396649-r14 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -175,48 +175,48 @@ - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:200 -INFO     Linting module: 'multiqc'__init__.py:204 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ -│                                           ╷                         ╷                            │ -│Module name                              │File path              │Test message              │ -│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ -│multiqc│modules/multiqc/main.nf│Conda update: │ -│││bioconda::multiqc 1.10 -> │ -│││1.13a│ -│                                           ╵                         ╵                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:200 +INFO     Linting module: 'multiqc'__init__.py:204 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ +│                                           ╷                         ╷                            │ +│Module name                              │File path              │Test message              │ +│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ +│multiqc│modules/multiqc/main.nf│Conda update: │ +│││bioconda::multiqc 1.10 -> │ +│││1.13a│ +│                                           ╵                         ╵                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 0344143c2b..156b7e5e2e 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-1311585603-matrix { + .terminal-3046389154-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1311585603-title { + .terminal-3046389154-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1311585603-r1 { fill: #c5c8c6 } -.terminal-1311585603-r2 { fill: #98a84b } -.terminal-1311585603-r3 { fill: #9a9b99 } -.terminal-1311585603-r4 { fill: #608ab1 } -.terminal-1311585603-r5 { fill: #d0b344 } -.terminal-1311585603-r6 { fill: #868887 } -.terminal-1311585603-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-1311585603-r8 { fill: #868887;font-style: italic; } + .terminal-3046389154-r1 { fill: #c5c8c6 } +.terminal-3046389154-r2 { fill: #98a84b } +.terminal-3046389154-r3 { fill: #9a9b99 } +.terminal-3046389154-r4 { fill: #608ab1 } +.terminal-3046389154-r5 { fill: #d0b344 } +.terminal-3046389154-r6 { fill: #868887 } +.terminal-3046389154-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3046389154-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                       list.py:124 - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name         ┃Repository     ┃Version SHA        ┃Message             ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftware… │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -│                      │                 │                     │ yaml files, add      │            │ -│                      │                 │                     │ yamllint config      │            │ -│                      │                 │                     │ (#1279)              │            │ -│ fastqc               │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -│                      │                 │                     │ yaml files, add      │            │ -│                      │                 │                     │ yamllint config      │            │ -│                      │                 │                     │ (#1279)              │            │ -│ multiqc              │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                       list.py:124 + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name         ┃Repository     ┃Version SHA        ┃Message             ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftware… │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ fastqc               │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +│                      │                 │                     │ yaml files, add      │            │ +│                      │                 │                     │ yamllint config      │            │ +│                      │                 │                     │ (#1279)              │            │ +│ multiqc              │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index dfbe22e11a..3a55feb8e7 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-1837409140-matrix { + .terminal-1756799955-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1837409140-title { + .terminal-1756799955-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1837409140-r1 { fill: #c5c8c6 } -.terminal-1837409140-r2 { fill: #98a84b } -.terminal-1837409140-r3 { fill: #9a9b99 } -.terminal-1837409140-r4 { fill: #608ab1 } -.terminal-1837409140-r5 { fill: #d0b344 } -.terminal-1837409140-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1837409140-r7 { fill: #868887 } -.terminal-1837409140-r8 { fill: #868887;font-style: italic; } + .terminal-1756799955-r1 { fill: #c5c8c6 } +.terminal-1756799955-r2 { fill: #98a84b } +.terminal-1756799955-r3 { fill: #9a9b99 } +.terminal-1756799955-r4 { fill: #608ab1 } +.terminal-1756799955-r5 { fill: #d0b344 } +.terminal-1756799955-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1756799955-r7 { fill: #868887 } +.terminal-1756799955-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):                                list.py:119 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO     Modules available from nf-core/modules (master):                                list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index 163f41a53d..cf8eec32a1 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -19,70 +19,70 @@ font-weight: 700; } - .terminal-1187224076-matrix { + .terminal-1435933291-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1187224076-title { + .terminal-1435933291-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1187224076-r1 { fill: #c5c8c6 } -.terminal-1187224076-r2 { fill: #98a84b } -.terminal-1187224076-r3 { fill: #9a9b99 } -.terminal-1187224076-r4 { fill: #608ab1 } -.terminal-1187224076-r5 { fill: #d0b344 } -.terminal-1187224076-r6 { fill: #868887 } -.terminal-1187224076-r7 { fill: #00823d;font-weight: bold } -.terminal-1187224076-r8 { fill: #68a0b3;font-weight: bold } + .terminal-1435933291-r1 { fill: #c5c8c6 } +.terminal-1435933291-r2 { fill: #98a84b } +.terminal-1435933291-r3 { fill: #9a9b99 } +.terminal-1435933291-r4 { fill: #608ab1 } +.terminal-1435933291-r5 { fill: #d0b344 } +.terminal-1435933291-r6 { fill: #868887 } +.terminal-1435933291-r7 { fill: #00823d;font-weight: bold } +.terminal-1435933291-r8 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -94,23 +94,23 @@ - + - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                              mulled.py:68 -INFO     Mulled container hash:                                                      __main__.py:828 -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                              mulled.py:68 +INFO     Mulled container hash:                                                      __main__.py:828 +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 0c7a4dd9f8..568a0a5a9f 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,119 +19,119 @@ font-weight: 700; } - .terminal-1724825069-matrix { + .terminal-2614607436-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1724825069-title { + .terminal-2614607436-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1724825069-r1 { fill: #c5c8c6 } -.terminal-1724825069-r2 { fill: #98a84b } -.terminal-1724825069-r3 { fill: #9a9b99 } -.terminal-1724825069-r4 { fill: #608ab1 } -.terminal-1724825069-r5 { fill: #d0b344 } -.terminal-1724825069-r6 { fill: #868887 } -.terminal-1724825069-r7 { fill: #ff2627 } -.terminal-1724825069-r8 { fill: #00823d } -.terminal-1724825069-r9 { fill: #ff2c7a;font-weight: bold } + .terminal-2614607436-r1 { fill: #c5c8c6 } +.terminal-2614607436-r2 { fill: #98a84b } +.terminal-2614607436-r3 { fill: #9a9b99 } +.terminal-2614607436-r4 { fill: #608ab1 } +.terminal-2614607436-r5 { fill: #d0b344 } +.terminal-2614607436-r6 { fill: #868887 } +.terminal-2614607436-r7 { fill: #ff2627 } +.terminal-2614607436-r8 { fill: #00823d } +.terminal-2614607436-r9 { fill: #ff2c7a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -143,39 +143,39 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Changes in module 'nf-core/modules/fastqc'modules_differ.py:252 -INFO    'modules/nf-core/modules/fastqc/meta.yml' is unchanged                modules_differ.py:257 -INFO     Changes in 'fastqc/main.nf':                                          modules_differ.py:266 - ---- modules/nf-core/modules/fastqc/main.nf -+++ modules/nf-core/modules/fastqc/main.nf -@@ -1,6 +1,6 @@ -process FASTQC {                                                                                   -    tag "$meta.id"                                                                                 --    label 'process_medium' -+    label 'process_low' - -    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)                                 -    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  - - -INFO     Patch file of 'nf-core/modules/fastqc' written to                              patch.py:115 -'modules/nf-core/modules/fastqc/fastqc.diff' + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO     Changes in module 'nf-core/modules/fastqc'modules_differ.py:252 +INFO    'modules/nf-core/modules/fastqc/meta.yml' is unchanged                modules_differ.py:257 +INFO     Changes in 'fastqc/main.nf':                                          modules_differ.py:266 + +--- modules/nf-core/modules/fastqc/main.nf ++++ modules/nf-core/modules/fastqc/main.nf +@@ -1,6 +1,6 @@ +process FASTQC {                                                                                   +    tag "$meta.id"                                                                                 +-    label 'process_medium' ++    label 'process_low' + +    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)                                 +    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  + + +INFO     Patch file of 'nf-core/modules/fastqc' written to                              patch.py:115 +'modules/nf-core/modules/fastqc/fastqc.diff' diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index bc67a96483..c2b680efe7 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-1051680509-matrix { + .terminal-4144914268-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1051680509-title { + .terminal-4144914268-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1051680509-r1 { fill: #c5c8c6 } -.terminal-1051680509-r2 { fill: #98a84b } -.terminal-1051680509-r3 { fill: #9a9b99 } -.terminal-1051680509-r4 { fill: #608ab1 } -.terminal-1051680509-r5 { fill: #d0b344 } -.terminal-1051680509-r6 { fill: #868887 } + .terminal-4144914268-r1 { fill: #c5c8c6 } +.terminal-4144914268-r2 { fill: #98a84b } +.terminal-4144914268-r3 { fill: #9a9b99 } +.terminal-4144914268-r4 { fill: #608ab1 } +.terminal-4144914268-r5 { fill: #d0b344 } +.terminal-4144914268-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Removing abacas                                                                remove.py:52 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO     Removing abacas                                                                remove.py:52 diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index b65ae7a6ad..22a073d723 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-2926910008-matrix { + .terminal-3641449111-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2926910008-title { + .terminal-3641449111-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2926910008-r1 { fill: #c5c8c6 } -.terminal-2926910008-r2 { fill: #98a84b } -.terminal-2926910008-r3 { fill: #9a9b99 } -.terminal-2926910008-r4 { fill: #608ab1 } -.terminal-2926910008-r5 { fill: #d0b344 } -.terminal-2926910008-r6 { fill: #868887 } + .terminal-3641449111-r1 { fill: #c5c8c6 } +.terminal-3641449111-r2 { fill: #98a84b } +.terminal-3641449111-r3 { fill: #9a9b99 } +.terminal-3641449111-r4 { fill: #608ab1 } +.terminal-3641449111-r5 { fill: #d0b344 } +.terminal-3641449111-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,23 +92,23 @@ - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view'module_test.py:184 + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view'module_test.py:184 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 080277cff0..e39d331663 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,77 +19,77 @@ font-weight: 700; } - .terminal-790941328-matrix { + .terminal-2235813615-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-790941328-title { + .terminal-2235813615-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-790941328-r1 { fill: #c5c8c6 } -.terminal-790941328-r2 { fill: #98a84b } -.terminal-790941328-r3 { fill: #9a9b99 } -.terminal-790941328-r4 { fill: #608ab1 } -.terminal-790941328-r5 { fill: #d0b344 } -.terminal-790941328-r6 { fill: #868887 } + .terminal-2235813615-r1 { fill: #c5c8c6 } +.terminal-2235813615-r2 { fill: #98a84b } +.terminal-2235813615-r3 { fill: #9a9b99 } +.terminal-2235813615-r4 { fill: #608ab1 } +.terminal-2235813615-r5 { fill: #d0b344 } +.terminal-2235813615-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +101,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO    'nf-core/modules/abacas' is already up to date                                update.py:160 -INFO     Updating 'nf-core/modules/custom/dumpsoftwareversions'update.py:516 -INFO     Updating 'nf-core/modules/fastqc'update.py:516 -INFO     Updating 'nf-core/modules/multiqc'update.py:516 -INFO     Updates complete ✨                                                           update.py:242 + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + + +INFO    'nf-core/modules/abacas' is already up to date                                update.py:160 +INFO     Updating 'nf-core/modules/custom/dumpsoftwareversions'update.py:516 +INFO     Updating 'nf-core/modules/fastqc'update.py:516 +INFO     Updating 'nf-core/modules/multiqc'update.py:516 +INFO     Updates complete ✨                                                           update.py:242 diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 87f27862ac..6845e1f07a 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-3538854618-matrix { + .terminal-3651117881-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3538854618-title { + .terminal-3651117881-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3538854618-r1 { fill: #c5c8c6 } -.terminal-3538854618-r2 { fill: #98a84b } -.terminal-3538854618-r3 { fill: #9a9b99 } -.terminal-3538854618-r4 { fill: #608ab1 } -.terminal-3538854618-r5 { fill: #d0b344 } -.terminal-3538854618-r6 { fill: #98a84b;font-weight: bold } -.terminal-3538854618-r7 { fill: #868887 } -.terminal-3538854618-r8 { fill: #868887;font-weight: bold } -.terminal-3538854618-r9 { fill: #4e707b;font-weight: bold } -.terminal-3538854618-r10 { fill: #68a0b3;font-weight: bold } + .terminal-3651117881-r1 { fill: #c5c8c6 } +.terminal-3651117881-r2 { fill: #98a84b } +.terminal-3651117881-r3 { fill: #9a9b99 } +.terminal-3651117881-r4 { fill: #608ab1 } +.terminal-3651117881-r5 { fill: #d0b344 } +.terminal-3651117881-r6 { fill: #98a84b;font-weight: bold } +.terminal-3651117881-r7 { fill: #868887 } +.terminal-3651117881-r8 { fill: #868887;font-weight: bold } +.terminal-3651117881-r9 { fill: #4e707b;font-weight: bold } +.terminal-3651117881-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 27 params)schema.py:95 -INFO     Writing schema with 28 params: './nextflow_schema.json'schema.py:173 + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 27 params)schema.py:95 +INFO     Writing schema with 28 params: './nextflow_schema.json'schema.py:173 diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index 0083a44941..47622daa2c 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-4207053170-matrix { + .terminal-3803351505-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4207053170-title { + .terminal-3803351505-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4207053170-r1 { fill: #c5c8c6 } -.terminal-4207053170-r2 { fill: #98a84b } -.terminal-4207053170-r3 { fill: #9a9b99 } -.terminal-4207053170-r4 { fill: #608ab1 } -.terminal-4207053170-r5 { fill: #d0b344 } -.terminal-4207053170-r6 { fill: #98a84b;font-weight: bold } -.terminal-4207053170-r7 { fill: #868887 } -.terminal-4207053170-r8 { fill: #868887;font-weight: bold } -.terminal-4207053170-r9 { fill: #4e707b;font-weight: bold } + .terminal-3803351505-r1 { fill: #c5c8c6 } +.terminal-3803351505-r2 { fill: #98a84b } +.terminal-3803351505-r3 { fill: #9a9b99 } +.terminal-3803351505-r4 { fill: #608ab1 } +.terminal-3803351505-r5 { fill: #d0b344 } +.terminal-3803351505-r6 { fill: #98a84b;font-weight: bold } +.terminal-3803351505-r7 { fill: #868887 } +.terminal-3803351505-r8 { fill: #868887;font-weight: bold } +.terminal-3803351505-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 2e089460a0..95f708fca7 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-229550800-matrix { + .terminal-735030063-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-229550800-title { + .terminal-735030063-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-229550800-r1 { fill: #c5c8c6 } -.terminal-229550800-r2 { fill: #98a84b } -.terminal-229550800-r3 { fill: #9a9b99 } -.terminal-229550800-r4 { fill: #608ab1 } -.terminal-229550800-r5 { fill: #d0b344 } -.terminal-229550800-r6 { fill: #98a84b;font-weight: bold } -.terminal-229550800-r7 { fill: #868887 } -.terminal-229550800-r8 { fill: #868887;font-weight: bold } -.terminal-229550800-r9 { fill: #4e707b;font-weight: bold } + .terminal-735030063-r1 { fill: #c5c8c6 } +.terminal-735030063-r2 { fill: #98a84b } +.terminal-735030063-r3 { fill: #9a9b99 } +.terminal-735030063-r4 { fill: #608ab1 } +.terminal-735030063-r5 { fill: #d0b344 } +.terminal-735030063-r6 { fill: #98a84b;font-weight: bold } +.terminal-735030063-r7 { fill: #868887 } +.terminal-735030063-r8 { fill: #868887;font-weight: bold } +.terminal-735030063-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 -INFO    [✓] Input parameters look validschema.py:213 + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 +INFO    [✓] Input parameters look validschema.py:213 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index ec1ac051e8..92e0acf675 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,73 +19,73 @@ font-weight: 700; } - .terminal-3145054114-matrix { + .terminal-3609966593-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3145054114-title { + .terminal-3609966593-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3145054114-r1 { fill: #c5c8c6 } -.terminal-3145054114-r2 { fill: #98a84b } -.terminal-3145054114-r3 { fill: #9a9b99 } -.terminal-3145054114-r4 { fill: #608ab1 } -.terminal-3145054114-r5 { fill: #d0b344 } -.terminal-3145054114-r6 { fill: #98729f } -.terminal-3145054114-r7 { fill: #ff2c7a } -.terminal-3145054114-r8 { fill: #868887 } + .terminal-3609966593-r1 { fill: #c5c8c6 } +.terminal-3609966593-r2 { fill: #98a84b } +.terminal-3609966593-r3 { fill: #9a9b99 } +.terminal-3609966593-r4 { fill: #608ab1 } +.terminal-3609966593-r5 { fill: #d0b344 } +.terminal-3609966593-r6 { fill: #98729f } +.terminal-3609966593-r7 { fill: #ff2c7a } +.terminal-3609966593-r8 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,24 +97,24 @@ - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 -INFO     Making a new template pipeline using pipeline variables                         sync.py:223 + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.1 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 +INFO     Making a new template pipeline using pipeline variables                         sync.py:223 From 95e3e0ee537ecff6d444ee17c1feb6cfc4896104 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 1 Sep 2022 15:49:43 +0200 Subject: [PATCH 057/854] bump to 2.6dev --- CHANGELOG.md | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8fc3f1003..89c8ac7834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # nf-core/tools: Changelog +## v2.6dev + +### Template + +### Linting + +### General + +### Modules + ## [v2.5.1 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5.1) - [2022-08-31] - Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)) diff --git a/setup.py b/setup.py index f3335db354..0759633a0b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.5.1" +version = "2.6dev" with open("README.md") as f: readme = f.read() From 76ee2610a0127260cf6ab4f8b86d03fc70ea2922 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Fri, 2 Sep 2022 06:43:38 +0200 Subject: [PATCH 058/854] Add teams messaging function --- .../assets/adaptivecard.json | 67 +++++++++++++++++++ .../lib/NfcoreTemplate.groovy | 55 +++++++++++++++ nf_core/pipeline-template/nextflow.config | 1 + .../pipeline-template/nextflow_schema.json | 7 ++ .../pipeline-template/workflows/pipeline.nf | 3 + 5 files changed, 133 insertions(+) create mode 100644 nf_core/pipeline-template/assets/adaptivecard.json diff --git a/nf_core/pipeline-template/assets/adaptivecard.json b/nf_core/pipeline-template/assets/adaptivecard.json new file mode 100644 index 0000000000..b818868d4b --- /dev/null +++ b/nf_core/pipeline-template/assets/adaptivecard.json @@ -0,0 +1,67 @@ +{ + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "contentUrl": null, + "content": { + "\$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "msteams": { + "width": "Full" + }, + "type": "AdaptiveCard", + "version": "1.2", + "body": [ + { + "type": "TextBlock", + "size": "Large", + "weight": "Bolder", + "color": "<% if (success) { %>Good<% } else { %>Attention<%} %>", + "text": "{{ name }} v${version} - ${runName}", + "wrap": true + }, + { + "type": "TextBlock", + "spacing": "None", + "text": "Completed at ${dateComplete} (duration: ${duration})", + "isSubtle": true, + "wrap": true + }, + { + "type": "TextBlock", + "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors. The full error message was: ${errorReport}.<% } %>", + "wrap": true + }, + { + "type": "TextBlock", + "text": "The command used to launch the workflow was as follows:", + "wrap": true + }, + { + "type": "TextBlock", + "text": "${commandLine}", + "isSubtle": true, + "wrap": true + } + ], + "actions": [ + { + "type": "Action.ShowCard", + "title": "Pipeline Configuration", + "card": { + "type": "AdaptiveCard", + "\$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "body": [ + { + "type": "FactSet", + "facts": [<% out << summary.collect{ k,v -> "{\"title\": \"$k\", \"value\" : \"$v\"}"}.join(",\n") %> + ] + } + ] + } + } + ] + } + } + ] +} diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 2894a6dd23..b6d981f831 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -145,6 +145,61 @@ class NfcoreTemplate { output_tf.withWriter { w -> w << email_txt } } + // + // Construct and send adaptive card + // https://adaptivecards.io + // + public static void adaptivecard(workflow, params, summary_params, projectDir, log, multiqc_report=[]) { + def hook_url = params.hook_url + + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) misc_fields['repository'] = workflow.repository + if (workflow.commitId) misc_fields['commitid'] = workflow.commitId + if (workflow.revision) misc_fields['revision'] = workflow.revision + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = workflow.manifest.version + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + def hf = new File("$projectDir/assets/adaptivecard.json") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection(); + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")); + def postRC = post.getResponseCode(); + if (! postRC.equals(200)) { + println(post.getErrorStream().getText()); + } + } + // // Print pipeline summary on completion // diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index fb9db8f03d..ba808b1d50 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -33,6 +33,7 @@ params { email_on_fail = null plaintext_email = false monochrome_logs = false + hook_url = null help = false validate_params = true show_hidden_params = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 5cd8ac489a..2be134ba08 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -213,6 +213,13 @@ "fa_icon": "fas fa-palette", "hidden": true }, + "hook_url": { + "type": "string", + "description": "Incoming hook URL for messaging service", + "fa_icon": "fas fa-people-group", + "help_text": "Incoming hook URL for messaging service. Currently, only MS Teams is supported.", + "hidden": true + }, "multiqc_config": { "type": "string", "description": "Custom config file to supply to MultiQC.", diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 97e80b7c3a..11d2ccf57d 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -114,6 +114,9 @@ workflow.onComplete { NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) } NfcoreTemplate.summary(workflow, params, log) + if (params.hook_url) { + NfcoreTemplate.adaptivecard(workflow, params, summary_params, projectDir, log, multiqc_report) + } } /* From 1446a30e763709e219a159cf48a84bb081b98410 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Fri, 2 Sep 2022 06:44:32 +0200 Subject: [PATCH 059/854] prettierignore --- nf_core/pipeline-template/.prettierignore | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore index d0e7ae5891..eb74a5741c 100644 --- a/nf_core/pipeline-template/.prettierignore +++ b/nf_core/pipeline-template/.prettierignore @@ -1,4 +1,5 @@ email_template.html +adaptivecard.json .nextflow* work/ data/ From 6ac8cbf8ac2db43d784b6f0f9330403fe9739ee2 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Fri, 2 Sep 2022 06:53:57 +0200 Subject: [PATCH 060/854] prettierignore --- .prettierignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.prettierignore b/.prettierignore index 10f3c7f4e7..5d06ce6ae8 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ email_template.html +adaptivecard.json docs/api/_build testing From 4137ca1cb73aeefd55b8cf62e68f8f9e4fb1869a Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Fri, 2 Sep 2022 09:51:50 +0200 Subject: [PATCH 061/854] Update nf_core/pipeline-template/lib/NfcoreTemplate.groovy --- nf_core/pipeline-template/lib/NfcoreTemplate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index b6d981f831..fda9f80270 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -196,7 +196,7 @@ class NfcoreTemplate { post.getOutputStream().write(json_message.getBytes("UTF-8")); def postRC = post.getResponseCode(); if (! postRC.equals(200)) { - println(post.getErrorStream().getText()); + log.warn(post.getErrorStream().getText()); } } From 70021aadcf2b1530cdd6f4b44cd9ce353d0ff0c8 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Fri, 2 Sep 2022 09:52:08 +0200 Subject: [PATCH 062/854] Update nf_core/pipeline-template/lib/NfcoreTemplate.groovy --- nf_core/pipeline-template/lib/NfcoreTemplate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index fda9f80270..8cd2cc1e9e 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -149,7 +149,7 @@ class NfcoreTemplate { // Construct and send adaptive card // https://adaptivecards.io // - public static void adaptivecard(workflow, params, summary_params, projectDir, log, multiqc_report=[]) { + public static void adaptivecard(workflow, params, summary_params, projectDir, log) { def hook_url = params.hook_url def summary = [:] From 882abf687170bd6b869c9da9938714eccfd31c0c Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Fri, 2 Sep 2022 11:34:13 +0200 Subject: [PATCH 063/854] Update nf_core/pipeline-template/workflows/pipeline.nf --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 11d2ccf57d..24f78e7746 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -115,7 +115,7 @@ workflow.onComplete { } NfcoreTemplate.summary(workflow, params, log) if (params.hook_url) { - NfcoreTemplate.adaptivecard(workflow, params, summary_params, projectDir, log, multiqc_report) + NfcoreTemplate.adaptivecard(workflow, params, summary_params, projectDir, log) } } From aee8e8294e45d8a860fa264d39f9d3ee81cf4bb0 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Fri, 2 Sep 2022 13:55:21 +0200 Subject: [PATCH 064/854] add azurebatch info --- nf_core/pipeline-template/docs/usage.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index aac1b9da5e..2125aeb830 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -246,6 +246,14 @@ See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). +## Azure Resource Requests + +To be used with the `azurebatch` profile by specifying the `-profile azurebatch`. +We recommend providing a compute `params.vm_type` of `Standard_E64_v3` VMs by default but these options can be changed if required. + +Note that the choice of VM size depends on your quota and the overall workload during the analysis. +For a thorough list, please refer the [Azure Sizes for virtual machines in Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes). + {% endif -%} ## Running in the background From 2034873a35e4bbb5e86eadf2e091c191c4112ce3 Mon Sep 17 00:00:00 2001 From: "Maxime U. Garcia" Date: Mon, 5 Sep 2022 11:06:57 +0200 Subject: [PATCH 065/854] Update nf_core/pipeline-template/docs/usage.md --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 2125aeb830..2c9969d04a 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -249,7 +249,7 @@ If you have any questions or issues please send us a message on [Slack](https:// ## Azure Resource Requests To be used with the `azurebatch` profile by specifying the `-profile azurebatch`. -We recommend providing a compute `params.vm_type` of `Standard_E64_v3` VMs by default but these options can be changed if required. +We recommend providing a compute `params.vm_type` of `Standard_D16_v3` VMs by default but these options can be changed if required. Note that the choice of VM size depends on your quota and the overall workload during the analysis. For a thorough list, please refer the [Azure Sizes for virtual machines in Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes). From bd9df71d9d7437cb759de612e07f7a0af795ac4e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 6 Sep 2022 10:37:28 +0200 Subject: [PATCH 066/854] Add actions/upload-artifact to the awstest workflows Exposes the new debug log file from the action to be downloaded --- CHANGELOG.md | 2 ++ nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 4 ++++ nf_core/pipeline-template/.github/workflows/awstest.yml | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c8ac7834..407f1e3ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file + ### Linting ### General diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index c37914276e..4b96fa80ee 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -28,3 +28,7 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } profiles: test_full,aws_tower + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 209cabded1..0f261fcb42 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -23,3 +23,7 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } profiles: test,aws_tower + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log From 364feeb9f7c2ab09894cb318f66c3c9f53b3d0b6 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Tue, 6 Sep 2022 16:29:29 +0200 Subject: [PATCH 067/854] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 407f1e3ed0..94fe689f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ### Linting ### General - +- Add function to enable chat notifications on teams, accompanied by `hook_url` param to enable it. ### Modules ## [v2.5.1 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5.1) - [2022-08-31] From e6003726d436ac015253a14ad956984446eeb200 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Tue, 6 Sep 2022 16:29:48 +0200 Subject: [PATCH 068/854] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94fe689f71..4ba6e6e229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ### Linting ### General -- Add function to enable chat notifications on teams, accompanied by `hook_url` param to enable it. +- Add function to enable chat notifications on MS Teams, accompanied by `hook_url` param to enable it. ### Modules ## [v2.5.1 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5.1) - [2022-08-31] From 2fcb80e363bcab3c00f86dc856429420d457f4cf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 6 Sep 2022 14:58:57 +0000 Subject: [PATCH 069/854] [automated] Fix linting with Prettier --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ba6e6e229..27b86f23ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ ### Linting ### General + - Add function to enable chat notifications on MS Teams, accompanied by `hook_url` param to enable it. + ### Modules ## [v2.5.1 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5.1) - [2022-08-31] From 26626bb25c7375d5b09cc02feb2e4544a96a4559 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 6 Sep 2022 14:21:00 +0200 Subject: [PATCH 070/854] generalized code, better error message and link update --- nf_core/pipeline-template/lib/Utils.groovy | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) mode change 100755 => 100644 nf_core/pipeline-template/lib/Utils.groovy diff --git a/nf_core/pipeline-template/lib/Utils.groovy b/nf_core/pipeline-template/lib/Utils.groovy old mode 100755 new mode 100644 index 28567bd70d..f5b2ba6c02 --- a/nf_core/pipeline-template/lib/Utils.groovy +++ b/nf_core/pipeline-template/lib/Utils.groovy @@ -21,19 +21,26 @@ class Utils { } // Check that all channels are present - def required_channels = ['conda-forge', 'bioconda', 'defaults'] - def conda_check_failed = !required_channels.every { ch -> ch in channels } + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order - conda_check_failed |= !(channels.indexOf('conda-forge') < channels.indexOf('bioconda')) - conda_check_failed |= !(channels.indexOf('bioconda') < channels.indexOf('defaults')) + def channel_priority_violation = false + def n = required_channels_in_order.size() + for (int i = 0; i < n - 1; i++) { + channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + } - if (conda_check_failed) { + if (channels_missing | channel_priority_violation) { log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " There is a problem with your Conda configuration!\n\n" + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/user/install.html#set-up-channels\n" + - " NB: The order of the channels matters!\n" + + " Please refer to https://bioconda.github.io/\n" + + " The observed channel order is \n" + + " ${channels}" + + " but the following channel order is required:\n" + + " ${required_channels}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" } } From 7017caa9f523c3655da9b897c17defdd9b1c6da3 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 6 Sep 2022 14:57:44 +0200 Subject: [PATCH 071/854] mend --- nf_core/pipeline-template/lib/Utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/Utils.groovy b/nf_core/pipeline-template/lib/Utils.groovy index f5b2ba6c02..7378f26f5e 100644 --- a/nf_core/pipeline-template/lib/Utils.groovy +++ b/nf_core/pipeline-template/lib/Utils.groovy @@ -40,7 +40,7 @@ class Utils { " The observed channel order is \n" + " ${channels}" + " but the following channel order is required:\n" + - " ${required_channels}\n" + + " ${required_channels_in_order}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" } } From 6029c9d5b302ae9ca5ae663d74ec326260b59e05 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 6 Sep 2022 17:37:40 +0200 Subject: [PATCH 072/854] better error message for bad conda channel setup --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27b86f23ce..7ee1a97beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file +- Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) ### Linting From 1712cc77af456efcfe9d08659e2d47cbf8d76b81 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 7 Sep 2022 08:54:28 +0200 Subject: [PATCH 073/854] better text formatting added missing newline --- nf_core/pipeline-template/lib/Utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/Utils.groovy b/nf_core/pipeline-template/lib/Utils.groovy index 7378f26f5e..8d030f4e84 100644 --- a/nf_core/pipeline-template/lib/Utils.groovy +++ b/nf_core/pipeline-template/lib/Utils.groovy @@ -38,7 +38,7 @@ class Utils { " You will need to set-up the conda-forge and bioconda channels correctly.\n" + " Please refer to https://bioconda.github.io/\n" + " The observed channel order is \n" + - " ${channels}" + + " ${channels}\n" + " but the following channel order is required:\n" + " ${required_channels_in_order}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" From 8fc5975caf6ab4d774da24180d3dbc2e2f7b8186 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 7 Sep 2022 10:57:46 +0200 Subject: [PATCH 074/854] Tweak confusing commit message --- .github/workflows/fix-linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 4409f1903b..674463f675 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -55,5 +55,5 @@ jobs: git config push.default upstream git add . git status - git commit -m "[automated] Fix linting with Prettier" + git commit -m "[automated] Fix code linting" git push From 74f65c68b6cd2c01db7a9808c58cdc6f3972ad40 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 12:19:25 +0200 Subject: [PATCH 075/854] fix typo in test class name --- tests/test_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_list.py b/tests/test_list.py index f71863cbca..2acef31a76 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -22,7 +22,7 @@ tmp_nxf_str = str(tmp_nxf) -class TestLint(unittest.TestCase): +class TestList(unittest.TestCase): """Class for list tests""" @mock.patch("subprocess.check_output") From 3c123ba745d98f8f53003276990eb34d8864f002 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 17:33:11 +0200 Subject: [PATCH 076/854] change to the working directory in a context --- tests/modules/lint.py | 10 ++++----- tests/modules/module_test.py | 40 ++++++++++++++++-------------------- tests/utils.py | 19 +++++++++++++++++ 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index d5793dfd05..df3a4cbe24 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -4,7 +4,7 @@ import nf_core.modules -from ..utils import GITLAB_URL +from ..utils import GITLAB_URL, set_wd from .patch import BISMARK_ALIGN, PATCH_BRANCH, setup_patch @@ -65,11 +65,9 @@ def test_modules_lint_patched_modules(self): # change temporarily working directory to the pipeline directory # to avoid error from try_apply_patch() during linting - wd_old = os.getcwd() - os.chdir(self.pipeline_dir) - module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) - module_lint.lint(print_results=False, all_modules=True) - os.chdir(wd_old) + with set_wd(self.pipeline_dir): + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) + module_lint.lint(print_results=False, all_modules=True) assert len(module_lint.failed) == 0 assert len(module_lint.passed) > 0 diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index ef955d0061..ce57312bf1 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -7,39 +7,35 @@ import nf_core.modules +from ..utils import set_wd + def test_modules_test_check_inputs(self): """Test the check_inputs() function - raise UserWarning because module doesn't exist""" - cwd = os.getcwd() - os.chdir(self.nfcore_modules) - meta_builder = nf_core.modules.ModulesTest("none", True, "") - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - os.chdir(cwd) + with set_wd(self.nfcore_modules): + meta_builder = nf_core.modules.ModulesTest("none", True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() assert "Cannot find directory" in str(excinfo.value) def test_modules_test_no_name_no_prompts(self): """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided.""" - cwd = os.getcwd() - os.chdir(self.nfcore_modules) - meta_builder = nf_core.modules.ModulesTest(None, True, "") - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - os.chdir(cwd) + with set_wd(self.nfcore_modules): + meta_builder = nf_core.modules.ModulesTest(None, True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() assert "Tool name not provided and prompts deactivated." in str(excinfo.value) def test_modules_test_no_installed_modules(self): """Test the check_inputs() function - raise UserWarning because installed modules were not found""" - cwd = os.getcwd() - os.chdir(self.nfcore_modules) - module_dir = Path(self.nfcore_modules, "modules") - shutil.rmtree(module_dir) - module_dir.mkdir() - meta_builder = nf_core.modules.ModulesTest(None, False, "") - meta_builder.repo_type = "modules" - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - os.chdir(cwd) + with set_wd(self.nfcore_modules): + module_dir = Path(self.nfcore_modules, "modules") + shutil.rmtree(module_dir) + module_dir.mkdir() + meta_builder = nf_core.modules.ModulesTest(None, False, "") + meta_builder.repo_type = "modules" + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() assert "No installed modules were found" in str(excinfo.value) diff --git a/tests/utils.py b/tests/utils.py index 03bfe272a0..a3e319b617 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,7 +5,10 @@ """ import functools +import os import tempfile +from contextlib import contextmanager +from pathlib import Path OLD_TRIMGALORE_SHA = "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" @@ -44,3 +47,19 @@ def wrapper(*args, **kwargs): return func(*args, tmpfile, **kwargs) return wrapper + + +@contextmanager +def set_wd(path: Path): + """Sets the working directory for this context. + + Arguments + --------- + + path : Path + Path to the working directory to be used iside this context. + """ + start_wd = Path().absolute() + os.chdir(path) + yield + os.chdir(start_wd) From 6f84c980aeae062157f9b063f7e9ba374822519d Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 17:39:01 +0200 Subject: [PATCH 077/854] use contextlib to temporarily set a working directry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27b86f23ce..6c0af1a1b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### General - Add function to enable chat notifications on MS Teams, accompanied by `hook_url` param to enable it. +- Use contextlib to temporarily change working directories ([#1819](https://github.com/nf-core/tools/pull/1819)) ### Modules From cf738225088c4931c887c96dc281915cd966d7f7 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Thu, 8 Sep 2022 08:31:34 +0200 Subject: [PATCH 078/854] make the context switch robust --- tests/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index a3e319b617..164df96c83 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -61,5 +61,7 @@ def set_wd(path: Path): """ start_wd = Path().absolute() os.chdir(path) - yield - os.chdir(start_wd) + try: + yield + finally: + os.chdir(start_wd) From 25d73bf4754a7260bc9152d4e59d061c54e41a5f Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 8 Sep 2022 09:20:12 +0200 Subject: [PATCH 079/854] Update MultiQC and insert config correctly --- nf_core/pipeline-template/modules.json | 12 ++++---- .../modules/nf-core/modules/multiqc/main.nf | 29 +++++++++++++++---- .../modules/nf-core/modules/multiqc/meta.yml | 8 +++++ .../pipeline-template/workflows/pipeline.nf | 13 +++++---- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 9c8d724aef..d17e160448 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -6,16 +6,16 @@ "git_url": "https://github.com/nf-core/modules.git", "modules": { "custom/dumpsoftwareversions": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" + "branch": "master", + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" }, "fastqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" + "branch": "master", + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" }, "multiqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" + "branch": "master", + "git_sha": "31166227d3dbc949f5403fb4c0ce2d65fb8107a4" } } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index 1264aac1eb..c8e6acee30 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -1,13 +1,15 @@ process MULTIQC { label 'process_medium' - conda (params.enable_conda ? 'bioconda::multiqc=1.12' : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13a' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.12--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13a--pyhdfd78af_1' : + 'quay.io/biocontainers/multiqc:1.13a--pyhdfd78af_1' }" input: - path multiqc_files + path multiqc_files, stageAs: "?/*" + path(multiqc_config) + path(multiqc_logo) output: path "*multiqc_report.html", emit: report @@ -20,8 +22,25 @@ process MULTIQC { script: def args = task.ext.args ?: '' + def config = multiqc_config ? "--config $multiqc_config" : '' """ - multiqc -f $args . + multiqc \\ + --force \\ + $config \\ + $args \\ + . + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS + """ + + stub: + """ + touch multiqc_data + touch multiqc_plots + touch multiqc_report.html cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml index 6fa891efc2..46cb92abe8 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml @@ -17,6 +17,14 @@ input: type: file description: | List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - multiqc_config: + type: file + description: Optional config yml for MultiQC + pattern: "*.{yml,yaml}" + - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" output: - report: type: file diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 24f78e7746..3bb18ed81d 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -23,8 +23,11 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() +ch_multiqc_config = params.multiqc_config ? file(params.multiqc_config, checkIfExists: true) : file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) + +// TODO nf-core: replace empty list with path to pipeline logo +ch_multiqc_logo = [] + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -90,14 +93,14 @@ workflow {{ short_name|upper }} { ch_workflow_summary = Channel.value(workflow_summary) ch_multiqc_files = Channel.empty() - ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) MULTIQC ( - ch_multiqc_files.collect() + ch_multiqc_files.collect(), + ch_multiqc_config, + ch_multiqc_logo ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) From 720f8ed38cd49a0017014f9ecb8b382fbc34e076 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 8 Sep 2022 09:21:40 +0200 Subject: [PATCH 080/854] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee1a97beb..4fdfd456e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) +- Update MultiQC module and correctly supply default MultiQC config file ### Linting From 3edfa5f05f42bd860b25ccf94ecb6387a88385b6 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 8 Sep 2022 09:23:09 +0200 Subject: [PATCH 081/854] Improve message re logo --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 3bb18ed81d..88b095ce66 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -25,7 +25,7 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ch_multiqc_config = params.multiqc_config ? file(params.multiqc_config, checkIfExists: true) : file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -// TODO nf-core: replace empty list with path to pipeline logo +// TODO nf-core: replace empty list with path to pipeline logo. Will require adding logo file name to MultiQC config to be displayed. ch_multiqc_logo = [] From 5fe2e8f5b41f57ed73e1af2179bff8539961974d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 8 Sep 2022 09:24:55 +0200 Subject: [PATCH 082/854] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fdfd456e3..07b1bb79cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) -- Update MultiQC module and correctly supply default MultiQC config file +- Update MultiQC module, update supplying MultiQC default and custom config files to module ### Linting From f19641ed9c9c5b9e0de2c47c2d7cb21e28bd46ae Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 8 Sep 2022 09:58:31 +0200 Subject: [PATCH 083/854] Try adding custom logo path parameter --- nf_core/pipeline-template/nextflow.config | 1 + nf_core/pipeline-template/nextflow_schema.json | 5 +++++ nf_core/pipeline-template/workflows/pipeline.nf | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ba808b1d50..1f5ebf13f3 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -23,6 +23,7 @@ params { // MultiQC options multiqc_config = null multiqc_title = null + multiqc_logo = "${projectDir}/assets/*_logo_light.png" max_multiqc_email_size = '25.MB' // Boilerplate options diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 2be134ba08..d3b77e5d50 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -39,6 +39,11 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" + }, + "multiqc_logo": { + "type": "string", + "description": "Path to custom logo for display in MultiQC report", + "fa_icon": "fas fa-image" } } }, diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 88b095ce66..8d48f37abc 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -23,10 +23,10 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -ch_multiqc_config = params.multiqc_config ? file(params.multiqc_config, checkIfExists: true) : file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) +ch_multiqc_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) // TODO nf-core: replace empty list with path to pipeline logo. Will require adding logo file name to MultiQC config to be displayed. -ch_multiqc_logo = [] +ch_multiqc_logo = file( params.multiqc_logo, checkIfExists: true ) /* From e3ec6dce10a241de5b3660d58852b75ee3bb89bb Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 8 Sep 2022 10:48:21 +0200 Subject: [PATCH 084/854] Remove MultiQC logo --- nf_core/pipeline-template/nextflow.config | 1 - nf_core/pipeline-template/nextflow_schema.json | 5 ----- nf_core/pipeline-template/workflows/pipeline.nf | 7 +------ 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 1f5ebf13f3..ba808b1d50 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -23,7 +23,6 @@ params { // MultiQC options multiqc_config = null multiqc_title = null - multiqc_logo = "${projectDir}/assets/*_logo_light.png" max_multiqc_email_size = '25.MB' // Boilerplate options diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index d3b77e5d50..2be134ba08 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -39,11 +39,6 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" - }, - "multiqc_logo": { - "type": "string", - "description": "Path to custom logo for display in MultiQC report", - "fa_icon": "fas fa-image" } } }, diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 8d48f37abc..a546ce50cb 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -25,10 +25,6 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ch_multiqc_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -// TODO nf-core: replace empty list with path to pipeline logo. Will require adding logo file name to MultiQC config to be displayed. -ch_multiqc_logo = file( params.multiqc_logo, checkIfExists: true ) - - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT LOCAL MODULES/SUBWORKFLOWS @@ -99,8 +95,7 @@ workflow {{ short_name|upper }} { MULTIQC ( ch_multiqc_files.collect(), - ch_multiqc_config, - ch_multiqc_logo + ch_multiqc_config ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) From e5dafc840c1298ea35adec30783c4872d75972d4 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 8 Sep 2022 10:59:51 +0200 Subject: [PATCH 085/854] Remove MultiQC logo from pipeline --- nf_core/pipeline-template/modules.json | 2 +- .../modules/nf-core/modules/multiqc/main.nf | 1 - .../modules/nf-core/modules/multiqc/meta.yml | 7 +++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index d17e160448..091b45d81d 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -15,7 +15,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "31166227d3dbc949f5403fb4c0ce2d65fb8107a4" + "git_sha": "8d2fedadff6b741a6bcf5144443035e1498b9640" } } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index c8e6acee30..c83ebf8587 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -9,7 +9,6 @@ process MULTIQC { input: path multiqc_files, stageAs: "?/*" path(multiqc_config) - path(multiqc_logo) output: path "*multiqc_report.html", emit: report diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml index 46cb92abe8..574f293efe 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml @@ -12,6 +12,7 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] + input: - multiqc_files: type: file @@ -21,10 +22,7 @@ input: type: file description: Optional config yml for MultiQC pattern: "*.{yml,yaml}" - - multiqc_logo: - type: file - description: Optional logo file for MultiQC - pattern: "*.{png}" + output: - report: type: file @@ -46,3 +44,4 @@ authors: - "@abhi18av" - "@bunop" - "@drpatelh" + - "@jfy133" From 72cff7a337b0c077f3bfa479172dfb9574f98c34 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Sep 2022 11:22:12 +0200 Subject: [PATCH 086/854] improve template customisation documentation --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32d0d8f557..7a66557952 100644 --- a/README.md +++ b/README.md @@ -401,7 +401,7 @@ Both options allow you to specify a custom pipeline prefix, as well as selecting The interactive prompts will guide you through the pipeline creation process. An example of a `template.yml` file is shown below. ```yaml -name: cool-pipe +name: coolpipe description: A cool pipeline author: me prefix: cool-pipes-company @@ -413,7 +413,12 @@ skip: - nf_core_configs ``` -This will create a pipeline called `cool-pipe` in the directory `cool-pipes-company-cool-pipe` with `me` as the author. It will exclude all files required for GitHub hosting of the pipeline, the GitHub CI from the pipeline, remove GitHub badges from the `README.md` file, remove pipeline options related to iGenomes and exclude `nf_core/configs` options. +This will create a pipeline called `coolpipe` in the directory `cool-pipes-company-coolpipe` (`-`) with `me` as the author. It will exclude all possible parts of the template: +- `github`: removed all files required for GitHub hosting of the pipeline. Specifically the `.github` folder and `.gitignore` file. +- `ci`: removes the GitHub continuous integration tests from the pipeline. Specifically the `.github/workflows/` folder. +- `github_badges`: removes GitHub badges from the `README.md` file. +- `igenomes`: removes pipeline options related to iGenomes. Including the `conf/igenomes.config` file and all references to it. +- `nf_core_configs`: excludes `nf_core/configs` repository options, which make multiple config profiles for various institutional clusters available. To run the pipeline creation silently (i.e. without any prompts) with the nf-core template, you can use the `--plain` option. From 6b6931aa0019ccc4a03a915fdbf452c518076981 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Sep 2022 11:27:53 +0200 Subject: [PATCH 087/854] explain prefix and update CHANGELOG --- CHANGELOG.md | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee1a97beb..5d3c4ea103 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) +- Improve template customisation documentation ([#1821](https://github.com/nf-core/tools/pull/1821)) ### Linting diff --git a/README.md b/README.md index 7a66557952..5ca789c0b8 100644 --- a/README.md +++ b/README.md @@ -397,7 +397,7 @@ Note that if the required arguments for `nf-core create` are not given, it will The `nf-core create` command comes with a number of options that allow you to customize the creation of a pipeline if you intend to not publish it as an nf-core pipeline. This can be done in two ways: by using interactive prompts, or by supplying a `template.yml` file using the `--template-yaml ` option. -Both options allow you to specify a custom pipeline prefix, as well as selecting parts of the template to be excluded during pipeline creation. +Both options allow you to specify a custom pipeline prefix to use instead of the common `nf-core`, as well as selecting parts of the template to be excluded during pipeline creation. The interactive prompts will guide you through the pipeline creation process. An example of a `template.yml` file is shown below. ```yaml From 5911070686b10a4a59c2b5c44565e28e15d0a22b Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 8 Sep 2022 09:30:02 +0000 Subject: [PATCH 088/854] [automated] Fix linting with Prettier --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5ca789c0b8..d89ba6086b 100644 --- a/README.md +++ b/README.md @@ -414,6 +414,7 @@ skip: ``` This will create a pipeline called `coolpipe` in the directory `cool-pipes-company-coolpipe` (`-`) with `me` as the author. It will exclude all possible parts of the template: + - `github`: removed all files required for GitHub hosting of the pipeline. Specifically the `.github` folder and `.gitignore` file. - `ci`: removes the GitHub continuous integration tests from the pipeline. Specifically the `.github/workflows/` folder. - `github_badges`: removes GitHub badges from the `README.md` file. From 19c22f25b4f7216bf7c5d474938f543e464b6396 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 8 Sep 2022 12:42:19 +0200 Subject: [PATCH 089/854] Re-add logo --- nf_core/pipeline-template/modules.json | 2 +- .../modules/nf-core/modules/multiqc/main.nf | 1 + .../modules/nf-core/modules/multiqc/meta.yml | 4 ++++ nf_core/pipeline-template/nextflow.config | 1 + nf_core/pipeline-template/nextflow_schema.json | 6 ++++++ nf_core/pipeline-template/workflows/pipeline.nf | 4 +++- 6 files changed, 16 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 091b45d81d..2a804dc1d0 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -15,7 +15,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "8d2fedadff6b741a6bcf5144443035e1498b9640" + "git_sha": "16eee433b87b303bda650131ac5a0b1ad725e166" } } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index c83ebf8587..c8e6acee30 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -9,6 +9,7 @@ process MULTIQC { input: path multiqc_files, stageAs: "?/*" path(multiqc_config) + path(multiqc_logo) output: path "*multiqc_report.html", emit: report diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml index 574f293efe..a1029f3366 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml @@ -22,6 +22,10 @@ input: type: file description: Optional config yml for MultiQC pattern: "*.{yml,yaml}" + - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" output: - report: diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ba808b1d50..c70f3a2e06 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -23,6 +23,7 @@ params { // MultiQC options multiqc_config = null multiqc_title = null + multiqc_logo = null max_multiqc_email_size = '25.MB' // Boilerplate options diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 2be134ba08..a4abd9b099 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -226,6 +226,12 @@ "fa_icon": "fas fa-cog", "hidden": true }, + "multiqc_logo": { + "type": "string", + "description": "Custom logo file to supply to MultiQC. File name must also be set in the MultiQC config file", + "fa_icon": "fas fa-image", + "hidden": true + }, "tracedir": { "type": "string", "description": "Directory to keep pipeline Nextflow logs and reports.", diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index a546ce50cb..605f1cb6c5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -24,6 +24,7 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample */ ch_multiqc_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) +ch_multiqc_logo = params.multiqc_logo ? file( params.multiqc_logo, checkIfExists: true ) : [] /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -95,7 +96,8 @@ workflow {{ short_name|upper }} { MULTIQC ( ch_multiqc_files.collect(), - ch_multiqc_config + ch_multiqc_config, + ch_multiqc_logo ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) From a9f2602eeb0300c97ce71806be81d899faeaeaa2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Sep 2022 15:14:12 +0200 Subject: [PATCH 090/854] check that name provided with template doesn't contain dashes --- nf_core/create.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/create.py b/nf_core/create.py index 1cd4fecba0..8c4d86761a 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -106,6 +106,10 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["name"] = self.get_param("name", name, template_yaml, template_yaml_path) param_dict["description"] = self.get_param("description", description, template_yaml, template_yaml_path) param_dict["author"] = self.get_param("author", author, template_yaml, template_yaml_path) + # Check that the pipeline name matches the requirements + if not re.match(r"^[a-z]+$", param_dict["name"]): + log.error("[red]Invalid workflow name: must be lowercase without punctuation.") + sys.exit(1) if "version" in template_yaml: if version is not None: From 31758174367ea758659fd066d7e4f79c83337376 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Sep 2022 15:15:07 +0200 Subject: [PATCH 091/854] ask for Name insteas of Workflow name with prompts --- nf_core/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 8c4d86761a..1bdc92f3d6 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -201,7 +201,7 @@ def get_param(self, param_name, passed_value, template_yaml, template_yaml_path) return passed_value def prompt_wf_name(self): - wf_name = questionary.text("Workflow name", style=nf_core.utils.nfcore_question_style).unsafe_ask() + wf_name = questionary.text("Name", style=nf_core.utils.nfcore_question_style).unsafe_ask() while not re.match(r"^[a-z]+$", wf_name): log.error("[red]Invalid workflow name: must be lowercase without punctuation.") wf_name = questionary.text( From e7ea34274862d6b11b1ca6f827cab520a7bee9aa Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Sep 2022 15:21:00 +0200 Subject: [PATCH 092/854] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee1a97beb..d418f15aa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) +- Check that the workflow name provided with a template doesn't contain dashes ([#1822](https://github.com/nf-core/tools/pull/1822)) +- Ask for `Name` instead of `Workflow name` when creating a pipeline with prompts ([#1822](https://github.com/nf-core/tools/pull/1822)) ### Linting From 3c95159fc5e321742c3d5a8d0cc92060b7e3cbef Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Sep 2022 15:42:15 +0200 Subject: [PATCH 093/854] fix checking wrong name --- nf_core/create.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 1bdc92f3d6..644fccf715 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -106,10 +106,6 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["name"] = self.get_param("name", name, template_yaml, template_yaml_path) param_dict["description"] = self.get_param("description", description, template_yaml, template_yaml_path) param_dict["author"] = self.get_param("author", author, template_yaml, template_yaml_path) - # Check that the pipeline name matches the requirements - if not re.match(r"^[a-z]+$", param_dict["name"]): - log.error("[red]Invalid workflow name: must be lowercase without punctuation.") - sys.exit(1) if "version" in template_yaml: if version is not None: @@ -166,6 +162,11 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version + # Check that the pipeline name matches the requirements + if not re.match(r"^[a-z]+$", param_dict["short_name"]): + log.error("[red]Invalid workflow name: must be lowercase without punctuation.") + sys.exit(1) + return param_dict, skip_paths def customize_template(self, template_areas): From d7c145f4ad03999fd289b9a6f2df4258922d418c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Sep 2022 16:12:09 +0200 Subject: [PATCH 094/854] remove underscore from testpipeline name in tests --- tests/test_launch.py | 2 +- tests/test_schema.py | 2 +- tests/test_sync.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_launch.py b/tests/test_launch.py index a438f98c2f..a7b42844d1 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -71,7 +71,7 @@ def test_make_pipeline_schema(self, tmp_path): """Create a workflow, but delete the schema file, then try to load it""" test_pipeline_dir = os.path.join(tmp_path, "wf") create_obj = nf_core.create.PipelineCreate( - "test_pipeline", "", "", outdir=test_pipeline_dir, no_git=True, plain=True + "testpipeline", "", "", outdir=test_pipeline_dir, no_git=True, plain=True ) create_obj.init_pipeline() os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) diff --git a/tests/test_schema.py b/tests/test_schema.py index 4f829875e7..2261c5ed97 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -31,7 +31,7 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.template_dir = os.path.join(self.tmp_dir, "wf") create_obj = nf_core.create.PipelineCreate( - "test_pipeline", "", "", outdir=self.template_dir, no_git=True, plain=True + "testpipeline", "", "", outdir=self.template_dir, no_git=True, plain=True ) create_obj.init_pipeline() diff --git a/tests/test_sync.py b/tests/test_sync.py index 2779f9e356..90a73590a4 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -21,7 +21,7 @@ class TestModules(unittest.TestCase): def setUp(self): """Create a new pipeline to test""" self.tmp_dir = tempfile.mkdtemp() - self.pipeline_dir = os.path.join(self.tmp_dir, "test_pipeline") + self.pipeline_dir = os.path.join(self.tmp_dir, "testpipeline") self.create_obj = nf_core.create.PipelineCreate( "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True ) From b9ff86ddfd3b47e364763498cba3c702bacd903f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 9 Sep 2022 09:57:48 +0200 Subject: [PATCH 095/854] Update nf_core/pipeline-template/assets/methods_description_template.yml Co-authored-by: Phil Ewels --- .../pipeline-template/assets/methods_description_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml index daa6fdf47c..f479bd0b31 100644 --- a/nf_core/pipeline-template/assets/methods_description_template.yml +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -1,5 +1,5 @@ id: "{{ name_noslash }}-methods-description" -description: "Suggested methods description text of pipeline usage." +description: "Suggested text and references to use when describing pipeline usage within the methods section of a publication." section_name: "{{ name }} Methods Description" section_href: "https://github.com/{{ name }}" plot_type: "html" From 3b8c3035e99e065fab6a90089100933cabd9a42f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 9 Sep 2022 08:01:52 +0000 Subject: [PATCH 096/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 101 ++++---- docs/images/nf-core-create.svg | 173 +++++++------- docs/images/nf-core-download.svg | 99 ++++---- docs/images/nf-core-launch-rnaseq.svg | 95 ++++---- docs/images/nf-core-licences.svg | 147 ++++++------ docs/images/nf-core-list-rna.svg | 129 ++++++----- docs/images/nf-core-list-stars.svg | 105 ++++----- docs/images/nf-core-list.svg | 109 ++++----- docs/images/nf-core-modules-bump-version.svg | 117 +++++----- docs/images/nf-core-modules-create.svg | 153 ++++++------ docs/images/nf-core-modules-info.svg | 231 ++++++++++--------- docs/images/nf-core-modules-install.svg | 93 ++++---- docs/images/nf-core-modules-lint.svg | 209 +++++++++-------- docs/images/nf-core-modules-list-local.svg | 133 +++++------ docs/images/nf-core-modules-list-remote.svg | 133 +++++------ docs/images/nf-core-modules-mulled.svg | 95 ++++---- docs/images/nf-core-modules-patch.svg | 161 ++++++------- docs/images/nf-core-modules-remove.svg | 83 +++---- docs/images/nf-core-modules-test.svg | 87 +++---- docs/images/nf-core-modules-update.svg | 99 ++++---- docs/images/nf-core-schema-build.svg | 95 ++++---- docs/images/nf-core-schema-lint.svg | 89 +++---- docs/images/nf-core-schema-validate.svg | 93 ++++---- docs/images/nf-core-sync.svg | 103 +++++---- 24 files changed, 1522 insertions(+), 1410 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index f42cb69e5f..e302e5ad91 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 -INFO     Updated version in 'nextflow.config'bump_version.py:164 - - version         = '1.0dev' - + version = '1.1' - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 +INFO     Updated version in 'nextflow.config'bump_version.py:164 + - version         = '1.0dev' + + version = '1.1' + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index d90cdfdaa7..47ac15e503 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - - - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  -big omics technique" -a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:226 -INFO     Initialising pipeline git repository                      create.py:518 -INFO     Done. Remember to add a remote and push to GitHub:        create.py:525 - cd  -/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core --nextbigthing - git remote add origin  -git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                    -INFO     This will also push your newly created dev branch and the create.py:531 -         TEMPLATE branch for syncing.                               -INFO    !!!!!! IMPORTANT !!!!!!create.py:217 - -If you are interested in adding your pipeline to the  -nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  -WRITING ANY CODE! - -Please read:  -https://nf-co.re/developers/adding_pipelines#join-the-com -munity + + + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next  +big omics technique" -a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:226 +INFO     Initialising pipeline git repository                      create.py:518 +INFO     Done. Remember to add a remote and push to GitHub:        create.py:525 + cd  +/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core +-nextbigthing + git remote add origin  +git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                    +INFO     This will also push your newly created dev branch and the create.py:531 +         TEMPLATE branch for syncing.                               +INFO    !!!!!! IMPORTANT !!!!!!create.py:217 + +If you are interested in adding your pipeline to the  +nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE  +WRITING ANY CODE! + +Please read:  +https://nf-co.re/developers/adding_pipelines#join-the-com +munity diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 5b479971ae..50335f3a6f 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq'download.py:158 -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                  download.py:161 -INFO     Downloading centralised configs from GitHub             download.py:165 + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     Saving 'nf-core/rnaseq'download.py:158 +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                  download.py:161 +INFO     Downloading centralised configs from GitHub             download.py:165 diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index f60ff94212..319de71823 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults   launch.py:131 -         overwritten by Nextflow config files or profiles           - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     NOTE: This tool ignores any pipeline parameter defaults   launch.py:131 +         overwritten by Nextflow config files or profiles           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index ced30525e1..7639066972 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                 licences.py:77 -INFO     Warning: This tool only prints licence information for   licences.py:98 -         the software tools packaged using conda.                  -INFO     The pipeline may use other software and dependencies not licences.py:99 -         described here.                                           -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     Fetching licence information for 8 tools                 licences.py:77 +INFO     Warning: This tool only prints licence information for   licences.py:98 +         the software tools packaged using conda.                  +INFO     The pipeline may use other software and dependencies not licences.py:99 +         described here.                                           +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 235b4a479a..db5e49cd52 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest┃ -┃Name       ┃Stars┃    Release┃    Released┃Last Pulled┃release?   ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ -│ rnafusion   │    76 │       2.1.0 │ 1 months ago │           - │ -           │ -│ smrnaseq    │    42 │       2.0.0 │ 3 months ago │           - │ -           │ -│ rnaseq      │   499 │       3.8.1 │ 3 months ago │           - │ -           │ -│ dualrnaseq  │     7 │       1.0.0 │  2 years ago │           - │ -           │ -│ circrna     │    19 │         dev │            - │           - │ -           │ -│ lncpipe     │    23 │         dev │            - │           - │ -           │ -│ scflow      │    12 │         dev │            - │           - │ -           │ -│ spatialtra… │    10 │         dev │            - │           - │ -           │ -└─────────────┴───────┴─────────────┴──────────────┴─────────────┴─────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest┃ +┃Name       ┃Stars┃    Release┃    Released┃Last Pulled┃release?   ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ +│ rnafusion   │    78 │       2.1.0 │ 2 months ago │           - │ -           │ +│ smrnaseq    │    43 │       2.0.0 │ 3 months ago │           - │ -           │ +│ rnaseq      │   503 │       3.8.1 │ 3 months ago │           - │ -           │ +│ dualrnaseq  │     7 │       1.0.0 │  2 years ago │           - │ -           │ +│ circrna     │    19 │         dev │            - │           - │ -           │ +│ lncpipe     │    22 │         dev │            - │           - │ -           │ +│ scflow      │    12 │         dev │            - │           - │ -           │ +│ spatialtra… │    10 │         dev │            - │           - │ -           │ +└─────────────┴───────┴─────────────┴──────────────┴─────────────┴─────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index cf4b23118e..74870d8f8b 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,89 @@ font-weight: 700; } - .terminal-2036675850-matrix { + .terminal-3543478864-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2036675850-title { + .terminal-3543478864-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2036675850-r1 { fill: #c5c8c6 } -.terminal-2036675850-r2 { fill: #98a84b } -.terminal-2036675850-r3 { fill: #9a9b99 } -.terminal-2036675850-r4 { fill: #608ab1 } -.terminal-2036675850-r5 { fill: #d0b344 } -.terminal-2036675850-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2036675850-r7 { fill: #868887 } -.terminal-2036675850-r8 { fill: #868887;font-style: italic; } + .terminal-3543478864-r1 { fill: #c5c8c6 } +.terminal-3543478864-r2 { fill: #98a84b } +.terminal-3543478864-r3 { fill: #9a9b99 } +.terminal-3543478864-r4 { fill: #608ab1 } +.terminal-3543478864-r5 { fill: #d0b344 } +.terminal-3543478864-r6 { fill: #d08442;font-weight: bold } +.terminal-3543478864-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3543478864-r8 { fill: #868887 } +.terminal-3543478864-r9 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +113,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ rnaseq      │   499 │       3.8.1 │    3 months │           - │ -            │ -│             │       │             │         ago │             │              │ -│ sarek       │   186 │       3.0.1 │  6 days ago │           - │ -            │ -│ chipseq     │   121 │       1.2.2 │ 1 years ago │           - │ -            │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest┃ +┃Name       ┃Stars┃    Release┃    Released┃Last Pulled┃release?   ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ +│ rnaseq      │   503 │       3.8.1 │ 3 months ago │           - │ -           │ +│ sarek       │   188 │       3.0.1 │  3 weeks ago │           - │ -           │ +│ chipseq     │   125 │       1.2.2 │  1 years ago │           - │ -           │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 5b0721e32e..19cadbc2a7 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,92 @@ font-weight: 700; } - .terminal-2123714818-matrix { + .terminal-2167229852-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2123714818-title { + .terminal-2167229852-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2123714818-r1 { fill: #c5c8c6 } -.terminal-2123714818-r2 { fill: #98a84b } -.terminal-2123714818-r3 { fill: #9a9b99 } -.terminal-2123714818-r4 { fill: #608ab1 } -.terminal-2123714818-r5 { fill: #d0b344 } -.terminal-2123714818-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2123714818-r7 { fill: #868887 } -.terminal-2123714818-r8 { fill: #868887;font-style: italic; } + .terminal-2167229852-r1 { fill: #c5c8c6 } +.terminal-2167229852-r2 { fill: #98a84b } +.terminal-2167229852-r3 { fill: #9a9b99 } +.terminal-2167229852-r4 { fill: #608ab1 } +.terminal-2167229852-r5 { fill: #d0b344 } +.terminal-2167229852-r6 { fill: #d08442;font-weight: bold } +.terminal-2167229852-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2167229852-r8 { fill: #868887 } +.terminal-2167229852-r9 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +116,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ -┃Pipeline   ┃┃     Latest┃┃┃Have latest ┃ -┃Name       ┃Stars┃    Release┃   Released┃Last Pulled┃release?    ┃ -┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ -│ sarek       │   186 │       3.0.1 │  6 days ago │           - │ -            │ -│ epitopepre… │    22 │       2.1.0 │ 3 weeks ago │           - │ -            │ -│ eager       │    70 │       2.4.5 │ 3 weeks ago │           - │ -            │ -│ viralrecon  │    74 │         2.5 │    1 months │           - │ -            │ -│             │       │             │         ago │             │              │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +┏━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ +┃Pipeline   ┃┃     Latest┃┃┃Have latest┃ +┃Name       ┃Stars┃    Release┃    Released┃Last Pulled┃release?   ┃ +┡━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ +│ ampliseq    │    93 │       2.4.0 │    yesterday │           - │ -           │ +│ mag         │   100 │       2.2.1 │  2 weeks ago │           - │ -           │ +│ sarek       │   188 │       3.0.1 │  3 weeks ago │           - │ -           │ +│ epitopepre… │    22 │       2.1.0 │ 1 months ago │           - │ -           │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 8ac0040588..0caa1f45af 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────╮ -│Module name                             │Update Message                   │ -├──────────────────────────────────────────┼───────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc │ -╰──────────────────────────────────────────┴───────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + + +╭──────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────╮ +│Module name                             │Update Message                   │ +├──────────────────────────────────────────┼───────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc │ +╰──────────────────────────────────────────┴───────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 69b382a0ad..abab34af2e 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low  ---meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Repository type: modulescreate.py:93 -INFO    Press enter to use default values (shown in brackets)or create.py:97 -type your own responses. ctrl+click underlined text to  -open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 -INFO     Using Docker container:                                   create.py:191 -'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1' -INFO     Using Singularity container:                              create.py:192 -'https://depot.galaxyproject.org/singularity/fastqc:0.11. -9--hdfd78af_1' -INFO     Created / edited following files:                         create.py:270 -           ./modules/fastqc/main.nf -           ./modules/fastqc/meta.yml -           ./tests/modules/fastqc/main.nf -           ./tests/modules/fastqc/test.yml -           ./tests/modules/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low  +--meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     Repository type: modulescreate.py:93 +INFO    Press enter to use default values (shown in brackets)or create.py:97 +type your own responses. ctrl+click underlined text to  +open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 +INFO     Using Docker container:                                   create.py:191 +'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1' +INFO     Using Singularity container:                              create.py:192 +'https://depot.galaxyproject.org/singularity/fastqc:0.11. +9--hdfd78af_1' +INFO     Created / edited following files:                         create.py:270 +           ./modules/fastqc/main.nf +           ./modules/fastqc/meta.yml +           ./tests/modules/fastqc/main.nf +           ./tests/modules/fastqc/test.yml +           ./tests/modules/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 69b4109b23..5b293e9c52 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - - - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                        │ -│ 🔧 Tools: abacas                                                             │ -│ 📖 Description: contiguate draft genome assembly                             │ -╰──────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                               ╷              -📥 Inputs        │Description                                    │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [│ -                  │id:'test', single_end:false ]                  │ -╶─────────────────┼───────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                 │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                           │*.{fasta,fa} -                  ╵                                               ╵              -                  ╷                                               ╷              -📤 Outputs       │Description                                    │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [│ -                  │id:'test', single_end:false ]                  │ -╶─────────────────┼───────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [          │ *.{abacas}* -                  │'test.abacas.bin', 'test.abacas.fasta',        │ -                  │'test.abacas.gaps', 'test.abacas.gaps.tab',    │ -                  │'test.abacas.nucmer.delta',                    │ -                  │'test.abacas.nucmer.filtered.delta',           │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',│ -                  │'test.abacas.unused.contigs.out',              │ -                  │'test.abacas.MULTIFASTA.fa' ]                  │ -╶─────────────────┼───────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions              │versions.yml -                  ╵                                               ╵              - - 💻  Installation command: nf-core modules install abacas - + + + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                        │ +│ 🔧 Tools: abacas                                                             │ +│ 📖 Description: contiguate draft genome assembly                             │ +╰──────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                               ╷              +📥 Inputs        │Description                                    │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [│ +                  │id:'test', single_end:false ]                  │ +╶─────────────────┼───────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                 │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                           │*.{fasta,fa} +                  ╵                                               ╵              +                  ╷                                               ╷              +📤 Outputs       │Description                                    │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [│ +                  │id:'test', single_end:false ]                  │ +╶─────────────────┼───────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [          │ *.{abacas}* +                  │'test.abacas.bin', 'test.abacas.fasta',        │ +                  │'test.abacas.gaps', 'test.abacas.gaps.tab',    │ +                  │'test.abacas.nucmer.delta',                    │ +                  │'test.abacas.nucmer.filtered.delta',           │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',│ +                  │'test.abacas.unused.contigs.out',              │ +                  │'test.abacas.MULTIFASTA.fa' ]                  │ +╶─────────────────┼───────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions              │versions.yml +                  ╵                                               ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 9a57d28b44..ddf4808be3 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Installing 'abacas'install.py:116 -INFO     Include statement: include { ABACAS } from               install.py:125 -'../modules/nf-core/modules/abacas/main' + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +INFO     Installing 'abacas'install.py:116 +INFO     Include statement: include { ABACAS } from               install.py:125 +'../modules/nf-core/modules/abacas/main' diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 8a4d1f6113..08cc7e2612 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:195 -INFO     Linting module: 'multiqc'__init__.py:199 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ -│                                           ╷                ╷                 │ -│Module name                              │File path     │Test message   │ -│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ -│multiqc│modules/multi…│Conda update: │ -│││bioconda::mult…│ -│││1.10 -> 1.13a│ -│                                           ╵                ╵                 │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +INFO     Linting modules repo: '.'__init__.py:195 +INFO     Linting module: 'multiqc'__init__.py:199 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────╮ +│                                           ╷                ╷                 │ +│Module name                              │File path     │Test message   │ +│╶──────────────────────────────────────────┼────────────────┼────────────────╴│ +│multiqc│modules/multi…│Conda update: │ +│││bioconda::mult…│ +│││1.10 -> 1.13│ +│                                           ╵                ╵                 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 9297ec424b..eb0fd4ff36 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,110 @@ font-weight: 700; } - .terminal-1989200096-matrix { + .terminal-2362882186-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1989200096-title { + .terminal-2362882186-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1989200096-r1 { fill: #c5c8c6 } -.terminal-1989200096-r2 { fill: #98a84b } -.terminal-1989200096-r3 { fill: #9a9b99 } -.terminal-1989200096-r4 { fill: #608ab1 } -.terminal-1989200096-r5 { fill: #d0b344 } -.terminal-1989200096-r6 { fill: #868887 } -.terminal-1989200096-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-1989200096-r8 { fill: #868887;font-style: italic; } + .terminal-2362882186-r1 { fill: #c5c8c6 } +.terminal-2362882186-r2 { fill: #98a84b } +.terminal-2362882186-r3 { fill: #9a9b99 } +.terminal-2362882186-r4 { fill: #608ab1 } +.terminal-2362882186-r5 { fill: #d0b344 } +.terminal-2362882186-r6 { fill: #d08442;font-weight: bold } +.terminal-2362882186-r7 { fill: #868887 } +.terminal-2362882186-r8 { fill: #c5c8c6;font-weight: bold } +.terminal-2362882186-r9 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +134,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules installed in '.':                                   list.py:124 - -┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ -┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -│               │                │               │ in yaml files, │            │ -│               │                │               │ add yamllint   │            │ -│               │                │               │ config (#1279) │            │ -│ multiqc       │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +INFO     Modules installed in '.':                                   list.py:124 + +┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name  ┃Repository    ┃Version SHA  ┃Message       ┃Date      ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumps… │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +│ fastqc        │ nf-core/modul… │ e745e167c102… │ Fix formatting │ 2022-02-15 │ +│               │                │               │ in yaml files, │            │ +│               │                │               │ add yamllint   │            │ +│               │                │               │ config (#1279) │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index e26dcf52b8..a2e2177d84 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,110 @@ font-weight: 700; } - .terminal-1871028241-matrix { + .terminal-2129569440-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1871028241-title { + .terminal-2129569440-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1871028241-r1 { fill: #c5c8c6 } -.terminal-1871028241-r2 { fill: #98a84b } -.terminal-1871028241-r3 { fill: #9a9b99 } -.terminal-1871028241-r4 { fill: #608ab1 } -.terminal-1871028241-r5 { fill: #d0b344 } -.terminal-1871028241-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1871028241-r7 { fill: #868887 } -.terminal-1871028241-r8 { fill: #868887;font-style: italic; } + .terminal-2129569440-r1 { fill: #c5c8c6 } +.terminal-2129569440-r2 { fill: #98a84b } +.terminal-2129569440-r3 { fill: #9a9b99 } +.terminal-2129569440-r4 { fill: #608ab1 } +.terminal-2129569440-r5 { fill: #d0b344 } +.terminal-2129569440-r6 { fill: #d08442;font-weight: bold } +.terminal-2129569440-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2129569440-r8 { fill: #868887 } +.terminal-2129569440-r9 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +134,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):            list.py:119 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +INFO     Modules available from nf-core/modules (master):            list.py:119 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index 0fcd819ed8..b1421cf115 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                          mulled.py:68 -INFO     Mulled container hash:                                  __main__.py:810 -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f -5ac4481e91f-0 + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     Found docker image on quay.io! ✨                          mulled.py:68 +INFO     Mulled container hash:                                  __main__.py:810 +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f +5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index ef6bf2f2ff..c56a5a1789 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Changes in module 'nf-core/modules/fastqc'modules_differ.py:252 -INFO    'modules/nf-core/modules/fastqc/meta.yml' is      modules_differ.py:257 -         unchanged                                          -INFO     Changes in 'fastqc/main.nf':                      modules_differ.py:266 - ---- modules/nf-core/modules/fastqc/main.nf -+++ modules/nf-core/modules/fastqc/main.nf -@@ -1,6 +1,6 @@ -process FASTQC {                                                               -    tag "$meta.id"                                                             --    label 'process_medium' -+    label 'process_low' - -    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)             -    container "${ workflow.containerEngine == 'singularity' && !task.ext.sing  - - -INFO     Patch file of 'nf-core/modules/fastqc' written to          patch.py:115 -'modules/nf-core/modules/fastqc/fastqc.diff' + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +INFO     Changes in module 'nf-core/modules/fastqc'modules_differ.py:252 +INFO    'modules/nf-core/modules/fastqc/meta.yml' is      modules_differ.py:257 +         unchanged                                          +INFO     Changes in 'fastqc/main.nf':                      modules_differ.py:266 + +--- modules/nf-core/modules/fastqc/main.nf ++++ modules/nf-core/modules/fastqc/main.nf +@@ -1,6 +1,6 @@ +process FASTQC {                                                               +    tag "$meta.id"                                                             +-    label 'process_medium' ++    label 'process_low' + +    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)             +    container "${ workflow.containerEngine == 'singularity' && !task.ext.sing  + + +INFO     Patch file of 'nf-core/modules/fastqc' written to          patch.py:115 +'modules/nf-core/modules/fastqc/fastqc.diff' diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index a8b3b7a17a..ca71c79a65 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO     Removing abacas                                            remove.py:52 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +INFO     Removing abacas                                            remove.py:52 diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index b08d0f3039..9d662ddd85 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -──────────────────────────────── samtools/view ───────────────────────────────── -INFO     Running pytest for module 'samtools/view'module_test.py:184 + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +──────────────────────────────── samtools/view ───────────────────────────────── +INFO     Running pytest for module 'samtools/view'module_test.py:184 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 559506e75e..40f7eb2403 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - - -INFO    'nf-core/modules/abacas' is already up to date            update.py:160 -INFO     Updating 'nf-core/modules/custom/dumpsoftwareversions'update.py:516 -INFO     Updating 'nf-core/modules/fastqc'update.py:516 -INFO     Updating 'nf-core/modules/multiqc'update.py:516 -INFO     Updates complete ✨                                       update.py:242 + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + + +INFO    'nf-core/modules/abacas' is already up to date            update.py:160 +INFO     Updating 'nf-core/modules/custom/dumpsoftwareversions'update.py:516 +INFO     Updating 'nf-core/modules/fastqc'update.py:516 +INFO     Updating 'nf-core/modules/multiqc'update.py:516 +INFO     Updates complete ✨                                       update.py:242 diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 67ba38633a..4c2be1800d 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 -INFO     Writing schema with 29 params: './nextflow_schema.json'schema.py:173 + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 +INFO     Writing schema with 29 params: './nextflow_schema.json'schema.py:173 diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index fcb19b9919..5177b75fe0 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 29 params)schema.py:95 + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 29 params)schema.py:95 diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 8ba9a2d519..9836f61564 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 -INFO    [✓] Input parameters look validschema.py:213 + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 +INFO    [✓] Input parameters look validschema.py:213 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 81f992bf71..3183d11846 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.dev0 - https://nf-co.re - - -INFO     Pipeline directory:                                          sync.py:95 -/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core-ne -xtbigthing -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 -INFO     Making a new template pipeline using pipeline variables     sync.py:223 + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.5.dev0 - https://nf-co.re +    There is a new version of nf-core/tools available! (2.5.1) + + +INFO     Pipeline directory:                                          sync.py:95 +/home/runner/work/nf-core-tools/nf-core-tools/tmp/nf-core-ne +xtbigthing +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                     sync.py:205 +INFO     Making a new template pipeline using pipeline variables     sync.py:223 From c74af0187da3ee5d3f568d682107d65e596c3cab Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 9 Sep 2022 10:36:07 +0200 Subject: [PATCH 097/854] Make DOI notice a condition --- .../pipeline-template/assets/methods_description_template.yml | 2 +- nf_core/pipeline-template/lib/WorkflowPipeline.groovy | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/assets/methods_description_template.yml b/nf_core/pipeline-template/assets/methods_description_template.yml index f479bd0b31..b2dc0a99c1 100644 --- a/nf_core/pipeline-template/assets/methods_description_template.yml +++ b/nf_core/pipeline-template/assets/methods_description_template.yml @@ -18,7 +18,7 @@ data: |
Notes:
    -
  • If present, make sure to replace the pipeline DOI with correct reference information.
    • + ${nodoi_text}
  • The command above does not include parameters contained in any configs or profiles that may have been used. Ensure the config file is also uploaded with your publication!
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 2cd2dceff4..252f127d80 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -53,7 +53,8 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { meta.workflow = run_workflow.toMap() meta["manifest_map"] = run_workflow.manifest.toMap() - meta["doi_text"] = meta.manifest_map.doi ? "(DOI: ${meta.manifest_map.doi})" : "" + meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" + meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " def methods_text = mqc_methods_yaml.text From 9aa0d91fa5c2ccc9f0b4a59e1bc39dea1fb03f0c Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 12 Sep 2022 11:34:34 +0200 Subject: [PATCH 098/854] test test utils --- tests/test_test_utils.py | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_test_utils.py diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py new file mode 100644 index 0000000000..ddf88ef74a --- /dev/null +++ b/tests/test_test_utils.py @@ -0,0 +1,50 @@ +import tempfile +from pathlib import Path + +import pytest + +from .utils import set_wd, with_temporary_file, with_temporary_folder + + +def test_with_temporary_file(): + @with_temporary_file + def tmp_file_exists(tmp_file): + assert Path(tmp_file.name).exists() + + tmp_file_exists() + + +def test_does_not_exist_after(): + tmp_file = with_temporary_file(lambda x: x.name)() + assert not Path(tmp_file).exists() + + +def test_with_temporary_folder(): + @with_temporary_folder + def tmp_folder_exists(tmp_folder): + assert Path(tmp_folder).exists() + + tmp_folder_exists() + + +def test_tmp_folder_does_not_exist_after(): + tmp_folder = with_temporary_folder(lambda x: x)() + assert not Path(tmp_folder).exists() + + +def test_set_wd(): + + with tempfile.TemporaryDirectory() as tmpdirname: + with set_wd(tmpdirname): + context_wd = Path().resolve() + assert context_wd == Path(tmpdirname).resolve() + assert context_wd != Path().resolve() + + +def test_set_wd_revert_on_raise(): + wd_before_context = Path().resolve() + with tempfile.TemporaryDirectory() as tmpdirname: + with pytest.raises(Exception): + with set_wd(tmpdirname): + raise Exception + assert wd_before_context == Path().resolve() From aef82ceec2c1a95d55daeb39866249d676005234 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Mon, 12 Sep 2022 16:21:23 +0200 Subject: [PATCH 099/854] Update MultiQC in pipeline template --- nf_core/pipeline-template/modules.json | 2 +- .../modules/nf-core/modules/multiqc/main.nf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 2a804dc1d0..a2e2430c29 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -15,7 +15,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "16eee433b87b303bda650131ac5a0b1ad725e166" + "git_sha": "5587389874dac9c9953a2ab6f01d49af81969492" } } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index c8e6acee30..d10dae694e 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_medium' - conda (params.enable_conda ? 'bioconda::multiqc=1.13a' : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13a--pyhdfd78af_1' : - 'quay.io/biocontainers/multiqc:1.13a--pyhdfd78af_1' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" From bc669b8d2fc3e416b089409690e3dde938b83e37 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 15:22:48 +0200 Subject: [PATCH 100/854] fix refresh rate propagation --- nf_core/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 9321ff9629..b6f17cc51c 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -339,7 +339,7 @@ def setup_requests_cachedir(): return config -def wait_cli_function(poll_func, poll_every=20): +def wait_cli_function(poll_func, refresh_per_second=20): """ Display a command-line spinner while calling a function repeatedly. @@ -347,14 +347,14 @@ def wait_cli_function(poll_func, poll_every=20): Arguments: poll_func (function): Function to call - poll_every (int): How many tenths of a second to wait between function calls. Default: 20. + refresh_per_second (int): Refresh this many times per second. Default: 20. Returns: None. Just sits in an infite loop until the function returns True. """ try: spinner = Spinner("dots2", "Use ctrl+c to stop waiting and force exit.") - with Live(spinner, refresh_per_second=20): + with Live(spinner, refresh_per_second=refresh_per_second): while True: if poll_func(): break From 4aa1be470a853a9c1d6f31169dd75d7da1bb5070 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:21:58 +0200 Subject: [PATCH 101/854] clean up unused argument module_lint_obj --- nf_core/lint/__init__.py | 2 +- nf_core/lint_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 7e61de3ac1..90e770f37b 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -124,7 +124,7 @@ def run_linting( lint_obj._print_results(show_passed) module_lint_obj._print_results(show_passed) nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj) - nf_core.lint_utils.print_fixes(lint_obj, module_lint_obj) + nf_core.lint_utils.print_fixes(lint_obj) # Save results to Markdown file if md_fn is not None: diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index ffb3bdf7b3..956dfea208 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -33,7 +33,7 @@ def print_joint_summary(lint_obj, module_lint_obj): console.print(table) -def print_fixes(lint_obj, module_lint_obj): +def print_fixes(lint_obj): """Prints available and applied fixes""" if len(lint_obj.could_fix): From 3ab94b27240437052da7195979d4dedb860b510e Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:23:13 +0200 Subject: [PATCH 102/854] fix line-to-long --- nf_core/lint_utils.py | 6 ++++-- nf_core/modules/test_yml_builder.py | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 956dfea208..b831644166 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -41,9 +41,11 @@ def print_fixes(lint_obj): "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}", " --fix ".join(lint_obj.could_fix) ) console.print( - f"\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n[blue] {fix_cmd}\n" + "\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n" + f"[blue] {fix_cmd}\n" ) if len(lint_obj.fix): console.print( - "Automatic fixes applied. Please check with 'git diff' and revert any changes you do not want with 'git checkout '." + "Automatic fixes applied. " + "Please check with 'git diff' and revert any changes you do not want with 'git checkout '." ) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index a47f7c352a..79a502604a 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -54,7 +54,8 @@ def run(self): """Run build steps""" if not self.no_prompts: log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + "[yellow]Press enter to use default values " + "[cyan bold](shown in brackets) [yellow]or type your own responses" ) self.check_inputs() self.scrape_workflow_entry_points() @@ -166,7 +167,10 @@ def build_single_test(self, entry_point): while ep_test["command"] == "": # Don't think we need the last `-c` flag, but keeping to avoid having to update 100s modules. # See https://github.com/nf-core/tools/issues/1562 - default_val = f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config -c ./tests/modules/{self.module_name}/nextflow.config" + default_val = ( + f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} " + f"-c ./tests/config/nextflow.config -c ./tests/modules/{self.module_name}/nextflow.config" + ) if self.no_prompts: ep_test["command"] = default_val else: From a2333cb8d840dbd9838bcf91f78328da7d86c086 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:23:45 +0200 Subject: [PATCH 103/854] use f-string --- nf_core/lint_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index b831644166..5495628e64 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -36,10 +36,10 @@ def print_joint_summary(lint_obj, module_lint_obj): def print_fixes(lint_obj): """Prints available and applied fixes""" - if len(lint_obj.could_fix): - fix_cmd = "nf-core lint {} --fix {}".format( - "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}", " --fix ".join(lint_obj.could_fix) - ) + if lint_obj.could_fix: + fix_flags = "".join([f" --fix {fix}" for fix in lint_obj.could_fix]) + wf_dir = "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}" + fix_cmd = f"nf-core lint {wf_dir} {fix_flags}" console.print( "\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n" f"[blue] {fix_cmd}\n" From accaaec2c1310225dbe20b68021eb283d8397bf6 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:25:13 +0200 Subject: [PATCH 104/854] clean up unused argument entry_point --- nf_core/modules/test_yml_builder.py | 4 ++-- tests/modules/create_test_yml.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 79a502604a..8254b3f761 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -192,7 +192,7 @@ def build_single_test(self, entry_point): ).strip() ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] - ep_test["files"] = self.get_md5_sums(entry_point, ep_test["command"]) + ep_test["files"] = self.get_md5_sums(ep_test["command"]) return ep_test @@ -254,7 +254,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): return test_files - def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repeat=None): + def get_md5_sums(self, command, results_dir=None, results_dir_repeat=None): """ Recursively go through directories and subdirectories and generate tuples of (, ) diff --git a/tests/modules/create_test_yml.py b/tests/modules/create_test_yml.py index dfb5fb5c6c..2f88b066a8 100644 --- a/tests/modules/create_test_yml.py +++ b/tests/modules/create_test_yml.py @@ -35,9 +35,7 @@ def test_modules_create_test_yml_get_md5(self, test_file_dir): meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", False, "./", False, True) with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") - test_files = meta_builder.get_md5_sums( - entry_point="dummy", command="dummy", results_dir=test_file_dir, results_dir_repeat=test_file_dir - ) + test_files = meta_builder.get_md5_sums(command="dummy", results_dir=test_file_dir, results_dir_repeat=test_file_dir) assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" From 7ee61f12c5a91b710cbcad643c3103eed38c3468 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:30:56 +0200 Subject: [PATCH 105/854] clean up unused argument hide_progress --- nf_core/modules/lint/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index dd1fd5b7d2..a35e10fde6 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -134,7 +134,6 @@ def lint( module=None, key=(), all_modules=False, - hide_progress=False, print_results=True, show_passed=False, local=False, From 750130431c688f8849774956a4fe3e6ec73a9e2f Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:42:39 +0200 Subject: [PATCH 106/854] fix line-to-long --- nf_core/modules/lint/main_nf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4b5327020f..1fb4a17a66 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -432,7 +432,8 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True ) log.debug( - f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, status_code: {response_new_container.status_code}" + f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, " + f"status_code: {response_new_container.status_code}" ) except (requests.exceptions.RequestException, sqlite3.InterfaceError) as e: log.debug(f"Unable to connect to url '{new_url}' due to error: {e}") From d136124e021ac713fe997e93dfaff140da755fa3 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:43:34 +0200 Subject: [PATCH 107/854] clean up unused argument self --- nf_core/modules/lint/main_nf.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 1fb4a17a66..0c449834c8 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -100,18 +100,18 @@ def main_nf(module_lint_object, module, fix_version, progress_bar): continue # Perform state-specific linting checks - if state == "process" and not _is_empty(module, l): + if state == "process" and not _is_empty(l): process_lines.append(l) - if state == "input" and not _is_empty(module, l): + if state == "input" and not _is_empty(l): inputs.extend(_parse_input(module, l)) - if state == "output" and not _is_empty(module, l): + if state == "output" and not _is_empty(l): outputs += _parse_output(module, l) outputs = list(set(outputs)) # remove duplicate 'meta's - if state == "when" and not _is_empty(module, l): + if state == "when" and not _is_empty(l): when_lines.append(l) - if state == "script" and not _is_empty(module, l): + if state == "script" and not _is_empty(l): script_lines.append(l) - if state == "shell" and not _is_empty(module, l): + if state == "shell" and not _is_empty(l): shell_lines.append(l) # Check that we have required sections @@ -391,7 +391,7 @@ def _parse_output(self, line): return output -def _is_empty(self, line): +def _is_empty(line): """Check whether a line is empty or a comment""" empty = False if line.strip().startswith("//"): From a7bc6de42b7a3fb4f7050d454ede67da1fdbe2b1 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:44:12 +0200 Subject: [PATCH 108/854] simplified condition --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0c449834c8..75d584a203 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -422,7 +422,7 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, build_type = _container_type(l) if build_type == "bioconda": new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) - elif build_type == "singularity" or build_type == "docker": + elif build_type in ("singularity", "docker"): # Check that the new url is valid new_url = re.search( "(?:['\"])(.+)(?:['\"])", re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line) From 49164ae51c405f7a28c1915f21b5edcd3aca8728 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 17:29:56 +0200 Subject: [PATCH 109/854] indicate unused self --- nf_core/modules/lint/module_deprecations.py | 2 +- nf_core/modules/lint/module_tests.py | 2 +- nf_core/modules/lint/module_todos.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/module_deprecations.py b/nf_core/modules/lint/module_deprecations.py index f7e8761c75..13ab2a583b 100644 --- a/nf_core/modules/lint/module_deprecations.py +++ b/nf_core/modules/lint/module_deprecations.py @@ -5,7 +5,7 @@ log = logging.getLogger(__name__) -def module_deprecations(module_lint_object, module): +def module_deprecations(_, module): """ Check that the modules are up to the latest nf-core standard """ diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b0c9fa0ee2..0b76acb944 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -9,7 +9,7 @@ log = logging.getLogger(__name__) -def module_tests(module_lint_object, module): +def module_tests(_, module): """ Lint the tests of a module in ``nf-core/modules`` diff --git a/nf_core/modules/lint/module_todos.py b/nf_core/modules/lint/module_todos.py index 90af44987e..82e7295eda 100644 --- a/nf_core/modules/lint/module_todos.py +++ b/nf_core/modules/lint/module_todos.py @@ -6,7 +6,7 @@ log = logging.getLogger(__name__) -def module_todos(module_lint_object, module): +def module_todos(_, module): """ Look for TODO statements in the module files From e8d5ae102a306efc1a9bb1791bb96850112a549f Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 15 Sep 2022 15:08:27 +0200 Subject: [PATCH 110/854] Update multiQC to allow extra configs --- nf_core/pipeline-template/modules.json | 2 +- .../modules/nf-core/modules/multiqc/main.nf | 5 ++++- .../modules/nf-core/modules/multiqc/meta.yml | 4 ++++ nf_core/pipeline-template/workflows/pipeline.nf | 6 ++++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index a2e2430c29..7533696f99 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -15,7 +15,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "5587389874dac9c9953a2ab6f01d49af81969492" + "git_sha": "4b1d4bf401d4cf65ebc4f604bc8c8e7551109db3" } } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index d10dae694e..698461d7b4 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -9,6 +9,7 @@ process MULTIQC { input: path multiqc_files, stageAs: "?/*" path(multiqc_config) + path(extra_multiqc_config) path(multiqc_logo) output: @@ -23,11 +24,13 @@ process MULTIQC { script: def args = task.ext.args ?: '' def config = multiqc_config ? "--config $multiqc_config" : '' + def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' """ multiqc \\ --force \\ - $config \\ $args \\ + $config \\ + $extra_config \\ . cat <<-END_VERSIONS > versions.yml diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml index a1029f3366..ebc29b279d 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml @@ -22,6 +22,10 @@ input: type: file description: Optional config yml for MultiQC pattern: "*.{yml,yaml}" + - extra_multiqc_config: + type: file + description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. + pattern: "*.{yml,yaml}" - multiqc_logo: type: file description: Optional logo file for MultiQC diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index b978ab81bc..7b307a7951 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -23,8 +23,9 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -ch_multiqc_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_logo = params.multiqc_logo ? file( params.multiqc_logo, checkIfExists: true ) : [] +ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) +ch_multiqc_config_custom = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : [] +ch_multiqc_logo = params.multiqc_logo ? file( params.multiqc_logo, checkIfExists: true ) : [] ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* @@ -102,6 +103,7 @@ workflow {{ short_name|upper }} { MULTIQC ( ch_multiqc_files.collect(), ch_multiqc_config, + ch_multiqc_config_custom, ch_multiqc_logo ) multiqc_report = MULTIQC.out.report.toList() From 23661044a466a16798fad742f10b8c9a52b531c2 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 15 Sep 2022 15:09:10 +0200 Subject: [PATCH 111/854] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab24c11301..f0a9588ce5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) -- Update MultiQC module, update supplying MultiQC default and custom config files to module +- Update MultiQC module, update supplying MultiQC default and custom config and logo files to module - Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) ### Linting From 4af66d7503cad92ebcc1389470b4b6b637ab83c5 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Sep 2022 15:13:32 +0200 Subject: [PATCH 112/854] Apply suggestions from code review --- nf_core/pipeline-template/workflows/pipeline.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 7b307a7951..b72a780d41 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -24,7 +24,7 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample */ ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_config_custom = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : [] +ch_multiqc_custom_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : [] ch_multiqc_logo = params.multiqc_logo ? file( params.multiqc_logo, checkIfExists: true ) : [] ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) @@ -103,7 +103,7 @@ workflow {{ short_name|upper }} { MULTIQC ( ch_multiqc_files.collect(), ch_multiqc_config, - ch_multiqc_config_custom, + ch_multiqc_custom_config, ch_multiqc_logo ) multiqc_report = MULTIQC.out.report.toList() From 583f33a61c982f11350d8b0bb1000485bc3c7aed Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Sep 2022 15:16:38 +0200 Subject: [PATCH 113/854] Replace [] with Channel.empty for consistency with previous version of tools --- nf_core/pipeline-template/workflows/pipeline.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index b72a780d41..83d49a327b 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -24,8 +24,8 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample */ ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : [] -ch_multiqc_logo = params.multiqc_logo ? file( params.multiqc_logo, checkIfExists: true ) : [] +ch_multiqc_custom_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : Channel.empty() +ch_multiqc_logo = params.multiqc_logo ? file( params.multiqc_logo, checkIfExists: true ) : Channel.empty() ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* From 7fb6c8fbaf9162f6e434abf788730dd4ad17b802 Mon Sep 17 00:00:00 2001 From: "Maxime U. Garcia" Date: Thu, 15 Sep 2022 17:11:20 +0200 Subject: [PATCH 114/854] Update CITATION.cff Ulysse is second first name, not familly name cf https://github.com/nf-core/tools/pull/1787 --- nf_core/pipeline-template/CITATION.cff | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/CITATION.cff b/nf_core/pipeline-template/CITATION.cff index 4533e2f28c..017666c018 100644 --- a/nf_core/pipeline-template/CITATION.cff +++ b/nf_core/pipeline-template/CITATION.cff @@ -13,8 +13,8 @@ authors: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen @@ -39,8 +39,8 @@ prefered-citation: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen From e793adf32cd475d397240cf8096a131c12a75afb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 20 Sep 2022 10:24:56 +0200 Subject: [PATCH 115/854] fix template spacing --- CHANGELOG.md | 1 + nf_core/pipeline-template/main.nf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0a9588ce5..c8bbc8ed94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) - Update MultiQC module, update supplying MultiQC default and custom config and logo files to module - Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) +- Fix template spacing modified by JINJA ([#1830](https://github.com/nf-core/tools/pull/1830)) ### Linting diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 539bcf2bf8..74dddd590d 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -4,7 +4,7 @@ {{ name }} ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/{{ name }} -{% if branded -%} +{% if branded %} Website: https://nf-co.re/{{ short_name }} Slack : https://nfcore.slack.com/channels/{{ short_name }} {% endif -%} From 2b00567fd5fd9d9aea932036e1bed1f41d0c5087 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 21 Sep 2022 09:08:59 +0200 Subject: [PATCH 116/854] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d89ba6086b..d0baab91eb 100644 --- a/README.md +++ b/README.md @@ -415,8 +415,8 @@ skip: This will create a pipeline called `coolpipe` in the directory `cool-pipes-company-coolpipe` (`-`) with `me` as the author. It will exclude all possible parts of the template: -- `github`: removed all files required for GitHub hosting of the pipeline. Specifically the `.github` folder and `.gitignore` file. -- `ci`: removes the GitHub continuous integration tests from the pipeline. Specifically the `.github/workflows/` folder. +- `github`: removed all files required for GitHub hosting of the pipeline. Specifically, the `.github` folder and `.gitignore` file. +- `ci`: removes the GitHub continuous integration tests from the pipeline. Specifically, the `.github/workflows/` folder. - `github_badges`: removes GitHub badges from the `README.md` file. - `igenomes`: removes pipeline options related to iGenomes. Including the `conf/igenomes.config` file and all references to it. - `nf_core_configs`: excludes `nf_core/configs` repository options, which make multiple config profiles for various institutional clusters available. From 19cd22a25d46587a661279f48e544c263df2df29 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 23 Sep 2022 14:30:48 +0200 Subject: [PATCH 117/854] add subworkflow template --- .../subworkflow-template/subworkflows/main.nf | 33 +++++++++++++++ .../subworkflows/meta.yml | 40 +++++++++++++++++++ nf_core/subworkflow-template/tests/main.nf | 18 +++++++++ .../tests/nextflow.config | 5 +++ nf_core/subworkflow-template/tests/test.yml | 14 +++++++ 5 files changed, 110 insertions(+) create mode 100644 nf_core/subworkflow-template/subworkflows/main.nf create mode 100644 nf_core/subworkflow-template/subworkflows/meta.yml create mode 100644 nf_core/subworkflow-template/tests/main.nf create mode 100644 nf_core/subworkflow-template/tests/nextflow.config create mode 100644 nf_core/subworkflow-template/tests/test.yml diff --git a/nf_core/subworkflow-template/subworkflows/main.nf b/nf_core/subworkflow-template/subworkflows/main.nf new file mode 100644 index 0000000000..47095a4697 --- /dev/null +++ b/nf_core/subworkflow-template/subworkflows/main.nf @@ -0,0 +1,33 @@ +// TODO nf-core: If in doubt look at other nf-core/subworkflows to see how we are doing things! :) +// https://github.com/nf-core/modules/tree/master/subworkflows +// You can also ask for help via your pull request or on the #subworkflows channel on the nf-core Slack workspace: +// https://nf-co.re/join +// TODO nf-core: A subworkflow SHOULD only import modules not other subworkflows + +workflow {{ subworkflow_name_underscore|upper }} { + + take: + // TODO nf-core: edit input (take) channels + ch_bam // channel: [ val(meta), [ bam ] ] + + main: + + ch_versions = Channel.empty() + + // TODO nf-core: substitute modules here for the modules of your subworkflow + + SAMTOOLS_SORT ( ch_bam ) + ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions.first()) + + SAMTOOLS_INDEX ( SAMTOOLS_SORT.out.bam ) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) + + emit: + // TODO nf-core: edit emitted channels + bam = SAMTOOLS_SORT.out.bam // channel: [ val(meta), [ bam ] ] + bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), [ bai ] ] + csi = SAMTOOLS_INDEX.out.csi // channel: [ val(meta), [ csi ] ] + + versions = ch_versions // channel: [ versions.yml ] +} + diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml new file mode 100644 index 0000000000..6cb5c48f15 --- /dev/null +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -0,0 +1,40 @@ +name: "{{ subworkflow_name_underscore }}" +## TODO nf-core: Add a description of the subworkflow and list keywords +description: Sort SAM/BAM/CRAM file +keywords: + - sort + - bam + - sam + - cram +## TODO nf-core: Add a list of the modules used in the subworkflow +modules: + - samtools/sort + - samtools/index +## TODO nf-core: List all of the variables used as input, including their types and descriptions +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test' ] + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: '*.{bam,cram,sam}' +## TODO nf-core: List all of the variables used as output, including their types and descriptions +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test' ] + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: '*.{bam,cram,sam}' + - versions: + type: file + description: File containing software versions + pattern: 'versions.yml' +authors: + - "{{ author }}" diff --git a/nf_core/subworkflow-template/tests/main.nf b/nf_core/subworkflow-template/tests/main.nf new file mode 100644 index 0000000000..1bd3ea0be9 --- /dev/null +++ b/nf_core/subworkflow-template/tests/main.nf @@ -0,0 +1,18 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { {{ subworkflow_name_underscore|upper }} } from '../../../{{ "../" if subtool else "" }}subworkflows/{{ subworkflow_dir }}/main.nf' + +workflow test_{{ subworkflow_name_underscore }} { + {% if has_meta %} + input = [ + [ id:'test' ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + {%- else %} + input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + {%- endif %} + + {{ subworkflow_name_underscore|upper }} ( input ) +} diff --git a/nf_core/subworkflow-template/tests/nextflow.config b/nf_core/subworkflow-template/tests/nextflow.config new file mode 100644 index 0000000000..8730f1c4b9 --- /dev/null +++ b/nf_core/subworkflow-template/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} diff --git a/nf_core/subworkflow-template/tests/test.yml b/nf_core/subworkflow-template/tests/test.yml new file mode 100644 index 0000000000..294c178f09 --- /dev/null +++ b/nf_core/subworkflow-template/tests/test.yml @@ -0,0 +1,14 @@ +## TODO nf-core: Please run the following command to build this file: +# nf-core subworkflows create-test-yml {{ subworkflow_name_underscore }} +- name: "{{ subworkflow_name_underscore }}" + command: nextflow run ./tests/modules/{{ subworkflow_dir }} -entry test_{{ subworkflow_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ tool_dir }}/nextflow.config + tags: + - "{{ tool }}" + # {%- if subtool %} + - "{{ tool }}/{{ subtool }}" + # {%- endif %} + files: + - path: "output/{{ tool }}/test.bam" + md5sum: e667c7caad0bc4b7ac383fd023c654fc + - path: output/{{ tool }}/versions.yml + md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b From 7445764eeb79c33060d8ecd7bef3aa6d9bfebfc5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 23 Sep 2022 14:35:26 +0200 Subject: [PATCH 118/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8bbc8ed94..0f49848176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Template +- Add template for subworkflows - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) - Update MultiQC module, update supplying MultiQC default and custom config and logo files to module From 76756b008ad360248897d6d0bd8cbc80fc4177d5 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 23 Sep 2022 12:37:32 +0000 Subject: [PATCH 119/854] [automated] Fix linting with Prettier --- nf_core/subworkflow-template/subworkflows/meta.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index 6cb5c48f15..28fdf80737 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -20,7 +20,7 @@ input: - bam: type: file description: BAM/CRAM/SAM file - pattern: '*.{bam,cram,sam}' + pattern: "*.{bam,cram,sam}" ## TODO nf-core: List all of the variables used as output, including their types and descriptions output: - meta: @@ -31,10 +31,10 @@ output: - bam: type: file description: Sorted BAM/CRAM/SAM file - pattern: '*.{bam,cram,sam}' + pattern: "*.{bam,cram,sam}" - versions: type: file description: File containing software versions - pattern: 'versions.yml' + pattern: "versions.yml" authors: - "{{ author }}" From a6de706186695061af903daef862f28eaaa9483e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 14:40:53 +0200 Subject: [PATCH 120/854] Add prettier to dockerimage for gitpod --- nf_core/gitpod/gitpod.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 7fbecc5e02..b8991ae823 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -31,6 +31,7 @@ RUN conda update -n base -c defaults conda && \ mamba=0.24.0 \ pip=22.1.2 \ black=22.6.0 \ + prettier=2.7.1 \ -n base && \ conda clean --all -f -y From 5689ff1558487f1fb10776452c609ed77c50d883 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 14:46:07 +0200 Subject: [PATCH 121/854] Add changelog for matthias to be happy --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8bbc8ed94..05274b446e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Add `actions/upload-artifact` step to the awstest workflows, to expose the debug log file +- Add `prettier` as a requirement to Gitpod Dockerimage - Bioconda incompatible conda channel setups now result in more informative error messages ([#1812](https://github.com/nf-core/tools/pull/1812)) - Update MultiQC module, update supplying MultiQC default and custom config and logo files to module - Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) From 0cf6c2c75d13884ce59b5caf7e793dd348cca77e Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 23 Sep 2022 14:46:31 +0200 Subject: [PATCH 122/854] add index files --- nf_core/subworkflow-template/subworkflows/meta.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index 6cb5c48f15..e42dffabf5 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -32,6 +32,14 @@ output: type: file description: Sorted BAM/CRAM/SAM file pattern: '*.{bam,cram,sam}' + - bai: + type: file + description: BAM/CRAM/SAM samtools index + pattern: '*.{bai,crai,sai}' + - csi: + type: file + description: CSI samtools index + pattern: '*.csi' - versions: type: file description: File containing software versions From 7004dd9df9238e4642598d9f0ebdcfcd4d092457 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 23 Sep 2022 14:50:45 +0200 Subject: [PATCH 123/854] update test.yml --- nf_core/subworkflow-template/tests/test.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nf_core/subworkflow-template/tests/test.yml b/nf_core/subworkflow-template/tests/test.yml index 294c178f09..d3650dafae 100644 --- a/nf_core/subworkflow-template/tests/test.yml +++ b/nf_core/subworkflow-template/tests/test.yml @@ -1,14 +1,12 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core subworkflows create-test-yml {{ subworkflow_name_underscore }} + - name: "{{ subworkflow_name_underscore }}" - command: nextflow run ./tests/modules/{{ subworkflow_dir }} -entry test_{{ subworkflow_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ tool_dir }}/nextflow.config + command: nextflow run ./tests/subworkflows/{{ subworkflow_dir }} -entry test_{{ subworkflow_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/subworkflows/{{ subworkflow_dir }}/nextflow.config tags: - - "{{ tool }}" - # {%- if subtool %} - - "{{ tool }}/{{ subtool }}" - # {%- endif %} + - "{{ subworkflow_name_underscore }}" files: - - path: "output/{{ tool }}/test.bam" + - path: "output/{{ subworkflow_name_underscore }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: output/{{ tool }}/versions.yml + - path: output/{{ subworkflow_name_underscore }}/versions.yml md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b From 6ed367bdc705f3be1e15c2985d8f25144d476196 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 23 Sep 2022 16:10:33 +0200 Subject: [PATCH 124/854] add `nf-core subworkflows create` --- nf_core/modules/create.py | 4 +- .../subworkflow-template/subworkflows/main.nf | 2 +- .../subworkflows/meta.yml | 2 +- nf_core/subworkflow-template/tests/main.nf | 6 +- nf_core/subworkflows/__init__.py | 1 + nf_core/subworkflows/create.py | 213 ++++++++++++++++++ 6 files changed, 221 insertions(+), 7 deletions(-) create mode 100644 nf_core/subworkflows/__init__.py create mode 100644 nf_core/subworkflows/create.py diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 9ea80cc7c5..b9886c295f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -136,7 +136,7 @@ def create(self): self.tool_name_underscore = self.tool_name.replace("/", "_") - # Check existance of directories early for fast-fail + # Check existence of directories early for fast-fail self.file_paths = self.get_module_dirs() # Try to find a bioconda package for 'tool' @@ -240,7 +240,7 @@ def create(self): "[violet]Will the module require a meta map of sample information?", default=True ) - # Create module template with cokiecutter + # Create module template with jinja self.render_template() if self.repo_type == "modules": diff --git a/nf_core/subworkflow-template/subworkflows/main.nf b/nf_core/subworkflow-template/subworkflows/main.nf index 47095a4697..eac79c8cff 100644 --- a/nf_core/subworkflow-template/subworkflows/main.nf +++ b/nf_core/subworkflow-template/subworkflows/main.nf @@ -4,7 +4,7 @@ // https://nf-co.re/join // TODO nf-core: A subworkflow SHOULD only import modules not other subworkflows -workflow {{ subworkflow_name_underscore|upper }} { +workflow {{ subworkflow_name|upper }} { take: // TODO nf-core: edit input (take) channels diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index 8cee246fa6..3db57b6fb1 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -1,4 +1,4 @@ -name: "{{ subworkflow_name_underscore }}" +name: "{{ subworkflow_name }}" ## TODO nf-core: Add a description of the subworkflow and list keywords description: Sort SAM/BAM/CRAM file keywords: diff --git a/nf_core/subworkflow-template/tests/main.nf b/nf_core/subworkflow-template/tests/main.nf index 1bd3ea0be9..0dd0b8f890 100644 --- a/nf_core/subworkflow-template/tests/main.nf +++ b/nf_core/subworkflow-template/tests/main.nf @@ -2,9 +2,9 @@ nextflow.enable.dsl = 2 -include { {{ subworkflow_name_underscore|upper }} } from '../../../{{ "../" if subtool else "" }}subworkflows/{{ subworkflow_dir }}/main.nf' +include { {{ subworkflow_name|upper }} } from '../../../{{ "../" if subtool else "" }}subworkflows/{{ subworkflow_dir }}/main.nf' -workflow test_{{ subworkflow_name_underscore }} { +workflow test_{{ subworkflow_name }} { {% if has_meta %} input = [ [ id:'test' ], // meta map @@ -14,5 +14,5 @@ workflow test_{{ subworkflow_name_underscore }} { input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) {%- endif %} - {{ subworkflow_name_underscore|upper }} ( input ) + {{ subworkflow_name|upper }} ( input ) } diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py new file mode 100644 index 0000000000..7e2aef35c7 --- /dev/null +++ b/nf_core/subworkflows/__init__.py @@ -0,0 +1 @@ +from .create import SubworkflowCreate diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py new file mode 100644 index 0000000000..764ea20c1c --- /dev/null +++ b/nf_core/subworkflows/create.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +""" +The SubworkflowCreate class handles generating of subworkflow templates +""" + +from __future__ import print_function + +import glob +import json +import logging +import os +import re +import subprocess + +import jinja2 +import questionary +import rich +import yaml +from packaging.version import parse as parse_version + +import nf_core +from nf_core.modules.module_utils import get_repo_type +import nf_core.utils + +log = logging.getLogger(__name__) + + +class SubworkflowCreate(object): + def __init__( + self, + directory=".", + subworkflow="", + author=None, + force=False, + ): + self.directory = directory + self.subworkflow = subworkflow + self.author = author + self.force_overwrite = force + self.file_paths = {} + + def create(self): + """ + Create a new subworkflow from the nf-core template. + + The subworkflow should be named as the main file type it operates on and a short description of the task performed + e.g bam_sort or bam_sort_samtools, respectively. + + If is a pipeline, this function creates a file called: + '/subworkflows/local/subworkflow_name.nf' + OR + '/subworkflows/local/subworkflow_name.nf' + + If is a clone of nf-core/modules, it creates or modifies the following files: + + subworkflows/subworkflow_name/ + * main.nf + * meta.yml + tests/subworkflows/subworkflow_name/ + * main.nf + * test.yml + * nextflow.config + tests/config/pytest_subworkflows.yml + + """ + + # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules + try: + self.directory, self.repo_type = get_repo_type(self.directory, self.repo_type) + except LookupError as e: + raise UserWarning(e) + log.info(f"Repository type: [blue]{self.repo_type}") + if self.directory != ".": + log.info(f"Base directory: '{self.directory}'") + + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " + "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." + ) + + # Collect module info via prompt if empty or invalid + if self.subworkflow is None: + self.subworkflow = "" + while self.subworkflow == "" or re.search(r"[^a-z\d/]", self.subworkflow) or self.subworkflow.count("/") > 0: + + # Check + auto-fix for invalid chacters + if re.search(r"[^a-z\d/]", self.subworkflow): + log.warning("Subworkflow name must be lower-case letters only, with no punctuation") + subworkflow_clean = re.sub(r"[^a-z\d/]", "", self.subworkflow.lower()) + if rich.prompt.Confirm.ask(f"[violet]Change '{self.subworkflow}' to '{subworkflow_clean}'?"): + self.subworkflow = subworkflow_clean + else: + self.subworkflow = "" + + # Prompt for new entry if we reset + if self.subworkflow == "": + self.subworkflow = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() + + # Determine the tool name + self.subworkflow_name = self.subworkflow + self.subworkflow_dir = self.subworkflow + + # Check existence of directories early for fast-fail + self.file_paths = self.get_subworkflow_dirs() + + # Prompt for GitHub username + # Try to guess the current user if `gh` is installed + author_default = None + try: + with open(os.devnull, "w") as devnull: + gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) + author_default = f"@{gh_auth_user['login']}" + except Exception as e: + log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") + + # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex + github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") + while self.author is None or not github_username_regex.match(self.author): + if self.author is not None and not github_username_regex.match(self.author): + log.warning("Does not look like a valid GitHub username (must start with an '@')!") + self.author = rich.prompt.Prompt.ask( + f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", + default=author_default, + ) + + # Create module template with jinja2 + self.render_template() + + if self.repo_type == "modules": + # Add entry to pytest_subworkflows.yml + try: + with open(os.path.join(self.directory, "tests", "config", "pytest_subworkflows.yml"), "r") as fh: + pytest_subworkflows_yml = yaml.safe_load(fh) + pytest_subworkflows_yml[self.subworkflow_name] = [ + f"subworkflows/{self.subworkflow}/**", + f"tests/subworkflows/{self.subworkflow}/**", + ] + pytest_subworkflows_yml = dict(sorted(pytest_subworkflows_yml.items())) + with open(os.path.join(self.directory, "tests", "config", "pytest_subworkflows.yml"), "w") as fh: + yaml.dump(pytest_subworkflows_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) + except FileNotFoundError as e: + raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") + + new_files = list(self.file_paths.values()) + if self.repo_type == "modules": + new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) + log.info("Created / edited following files:\n " + "\n ".join(new_files)) + + def render_template(self): + """ + Create new subworkflow files with Jinja2. + """ + # Run jinja2 for each file in the template folder + env = jinja2.Environment( + loader=jinja2.PackageLoader("nf_core", "subworkflow-template"), keep_trailing_newline=True + ) + for template_fn, dest_fn in self.file_paths.items(): + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + object_attrs = vars(self) + object_attrs["nf_core_version"] = nf_core.__version__ + rendered_output = j_template.render(object_attrs) + + # Write output to the target file + os.makedirs(os.path.dirname(dest_fn), exist_ok=True) + with open(dest_fn, "w") as fh: + log.debug(f"Writing output to: '{dest_fn}'") + fh.write(rendered_output) + + # Mirror file permissions + template_stat = os.stat( + os.path.join(os.path.dirname(nf_core.__file__), "subworkflow-template", template_fn) + ) + os.chmod(dest_fn, template_stat.st_mode) + + def get_subworkflow_dirs(self): + """Given a directory and a subworkflow, set the file paths and check if they already exist + + Returns dict: keys are relative paths to template files, vals are target paths. + """ + + file_paths = {} + + if self.repo_type == "pipeline": + local_subworkflow_dir = os.path.join(self.directory, "subworkflows", "local") + + # Check whether subworkflow file already exists + subworkflow_file = os.path.join(local_subworkflow_dir, f"{self.subworkflow_name}.nf") + if os.path.exists(subworkflow_file) and not self.force_overwrite: + raise UserWarning(f"Subworkflow file exists already: '{subworkflow_file}'. Use '--force' to overwrite") + + # Set file paths + file_paths[os.path.join("subworkflows", f"{self.subworkflow_name}.nf")] = subworkflow_file + + if self.repo_type == "modules": + subworkflow_path = os.path.join(self.directory, "subworkflows", self.subworkflow_dir) + test_dir = os.path.join(self.directory, "tests", "subworkflows", self.subworkflow_dir) + + # Check if module directories exist already + if os.path.exists(subworkflow_path) and not self.force_overwrite: + raise UserWarning(f"Subworkflow directory exists: '{subworkflow_path}'. Use '--force' to overwrite") + + if os.path.exists(test_dir) and not self.force_overwrite: + raise UserWarning(f"Subworkflow test directory exists: '{test_dir}'. Use '--force' to overwrite") + + # Set file paths + file_paths[os.path.join("subworkflows", "main.nf")] = os.path.join(subworkflow_path, "main.nf") + file_paths[os.path.join("subworkflows", "meta.yml")] = os.path.join(subworkflow_path, "meta.yml") + file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") + file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") + file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") + + return file_paths From 85f46263c675a5b5492dce7b612c9445130569de Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 23 Sep 2022 17:17:23 +0200 Subject: [PATCH 125/854] subworkflow create command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/__main__.py | 69 +++++++++++++++++++++++++++++++++- nf_core/subworkflows/create.py | 22 ++++++----- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 39cd1390ab..073ff333c4 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -19,6 +19,7 @@ import nf_core.lint import nf_core.list import nf_core.modules +import nf_core.subworkflows import nf_core.schema import nf_core.sync import nf_core.utils @@ -54,6 +55,12 @@ "commands": ["create", "create-test-yml", "lint", "bump-versions", "mulled", "test"], }, ], + "nf-core subworkflows": [ + { + "name": "Developing new subworkflows", + "commands": ["create"], + }, + ] } click.rich_click.OPTION_GROUPS = { "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}], @@ -89,7 +96,7 @@ def run_nf_core(): log.debug(f"Could not check latest version: {e}") stderr.print("\n") - # Lanch the click cli + # Launch the click cli nf_core_cli(auto_envvar_prefix="NFCORE") @@ -382,6 +389,36 @@ def modules(ctx, git_remote, branch, no_pull): ctx.obj["modules_repo_branch"] = branch ctx.obj["modules_repo_no_pull"] = no_pull +#nf-core subworkflows click command +@nf_core_cli.group() +@click.option( + "-g", + "--git-remote", + type=str, + default=nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE, + help="Remote git repo to fetch files from", +) +@click.option("-b", "--branch", type=str, default=None, help="Branch of git repository hosting modules.") +@click.option( + "-N", + "--no-pull", + is_flag=True, + default=False, + help="Do not pull in latest changes to local clone of modules repository.", +) +@click.pass_context +def subworkflows(ctx, git_remote, branch, no_pull): + """ + Commands to manage Nextflow DSL2 subworkflows (tool wrappers). + """ + # ensure that ctx.obj exists and is a dict (in case `cli()` is called + # by means other than the `if` block below) + ctx.ensure_object(dict) + + # Place the arguments in a context object + ctx.obj["subworkflows_repo_url"] = git_remote + ctx.obj["subworkflows_repo_branch"] = branch + ctx.obj["subworkflows_repo_no_pull"] = no_pull # nf-core modules list subcommands @modules.group() @@ -848,6 +885,36 @@ def test_module(ctx, tool, no_prompts, pytest_args): log.critical(e) sys.exit(1) +# nf-core subworkflows create +@subworkflows.command("create") +@click.pass_context +@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") +@click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") +def create_subworkflow(ctx, subworkflow, dir, author, force): + """ + Create a new subworkflow from the nf-core template. + + If the specified directory is a pipeline, this function creates a file called + 'subworkflows/local/.nf' + + If the specified directory is a clone of nf-core/modules, it creates or modifies files + in 'subworkflows/', 'tests/subworkflows' and 'tests/config/pytest_modules.yml' + """ + + # Run function + try: + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( + dir, subworkflow, author, force + ) + subworkflow_create.create() + except UserWarning as e: + log.critical(e) + sys.exit(1) + except LookupError as e: + log.error(e) + sys.exit(1) # nf-core schema subcommands @nf_core_cli.group() diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 764ea20c1c..661886e45a 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -32,12 +32,14 @@ def __init__( subworkflow="", author=None, force=False, + repo_type=None, ): self.directory = directory self.subworkflow = subworkflow self.author = author self.force_overwrite = force self.file_paths = {} + self.repo_type = repo_type def create(self): """ @@ -60,7 +62,7 @@ def create(self): * main.nf * test.yml * nextflow.config - tests/config/pytest_subworkflows.yml + tests/config/pytest_modules.yml """ @@ -127,17 +129,17 @@ def create(self): self.render_template() if self.repo_type == "modules": - # Add entry to pytest_subworkflows.yml + # Add entry to pytest_modules.yml try: - with open(os.path.join(self.directory, "tests", "config", "pytest_subworkflows.yml"), "r") as fh: - pytest_subworkflows_yml = yaml.safe_load(fh) - pytest_subworkflows_yml[self.subworkflow_name] = [ + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: + pytest_modules_yml = yaml.safe_load(fh) + pytest_modules_yml[self.subworkflow] = [ f"subworkflows/{self.subworkflow}/**", f"tests/subworkflows/{self.subworkflow}/**", ] - pytest_subworkflows_yml = dict(sorted(pytest_subworkflows_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_subworkflows.yml"), "w") as fh: - yaml.dump(pytest_subworkflows_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) + pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: + yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) except FileNotFoundError as e: raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") @@ -193,8 +195,8 @@ def get_subworkflow_dirs(self): file_paths[os.path.join("subworkflows", f"{self.subworkflow_name}.nf")] = subworkflow_file if self.repo_type == "modules": - subworkflow_path = os.path.join(self.directory, "subworkflows", self.subworkflow_dir) - test_dir = os.path.join(self.directory, "tests", "subworkflows", self.subworkflow_dir) + subworkflow_path = os.path.join(self.directory, "subworkflows", "nf-core", self.subworkflow_dir) + test_dir = os.path.join(self.directory, "tests", "subworkflows", "nf-core", self.subworkflow_dir) # Check if module directories exist already if os.path.exists(subworkflow_path) and not self.force_overwrite: From e1dc0de15428aa0e909fdc1533f3baeb7a0d3475 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 23 Sep 2022 17:22:50 +0200 Subject: [PATCH 126/854] run isort --- nf_core/__main__.py | 2 +- nf_core/subworkflows/create.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 073ff333c4..4852224c88 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -19,8 +19,8 @@ import nf_core.lint import nf_core.list import nf_core.modules -import nf_core.subworkflows import nf_core.schema +import nf_core.subworkflows import nf_core.sync import nf_core.utils diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 661886e45a..1a33284a27 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -19,8 +19,8 @@ from packaging.version import parse as parse_version import nf_core -from nf_core.modules.module_utils import get_repo_type import nf_core.utils +from nf_core.modules.module_utils import get_repo_type log = logging.getLogger(__name__) From 21d3452d5d153a5b132d59b7eb721371d8bea531 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 23 Sep 2022 17:25:12 +0200 Subject: [PATCH 127/854] fix black linting --- nf_core/__main__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 4852224c88..8223ac7e61 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -60,7 +60,7 @@ "name": "Developing new subworkflows", "commands": ["create"], }, - ] + ], } click.rich_click.OPTION_GROUPS = { "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}], @@ -389,7 +389,8 @@ def modules(ctx, git_remote, branch, no_pull): ctx.obj["modules_repo_branch"] = branch ctx.obj["modules_repo_no_pull"] = no_pull -#nf-core subworkflows click command + +# nf-core subworkflows click command @nf_core_cli.group() @click.option( "-g", @@ -420,6 +421,7 @@ def subworkflows(ctx, git_remote, branch, no_pull): ctx.obj["subworkflows_repo_branch"] = branch ctx.obj["subworkflows_repo_no_pull"] = no_pull + # nf-core modules list subcommands @modules.group() @click.pass_context @@ -885,6 +887,7 @@ def test_module(ctx, tool, no_prompts, pytest_args): log.critical(e) sys.exit(1) + # nf-core subworkflows create @subworkflows.command("create") @click.pass_context @@ -905,9 +908,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): # Run function try: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate( - dir, subworkflow, author, force - ) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force) subworkflow_create.create() except UserWarning as e: log.critical(e) @@ -916,6 +917,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): log.error(e) sys.exit(1) + # nf-core schema subcommands @nf_core_cli.group() def schema(): From a1f743d4f9aafd6c3d3e3a2f994cda8bce195c0a Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 23 Sep 2022 18:13:56 +0200 Subject: [PATCH 128/854] add subworkflows tests --- tests/subworkflows/__init__.py | 0 tests/subworkflows/create.py | 35 +++++++++++++++++++ tests/test_subworkflows.py | 64 ++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 tests/subworkflows/__init__.py create mode 100644 tests/subworkflows/create.py create mode 100644 tests/test_subworkflows.py diff --git a/tests/subworkflows/__init__.py b/tests/subworkflows/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py new file mode 100644 index 0000000000..1b838eec1d --- /dev/null +++ b/tests/subworkflows/create.py @@ -0,0 +1,35 @@ +import os + +import pytest + +import nf_core.subworkflows + + +def test_subworkflows_create_succeed(self): + """Succeed at creating a subworkflow from the template inside a pipeline""" + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( + self.pipeline_dir, "test_subworkflow", "@author", True + ) + subworkflow_create.create() + assert os.path.exists(os.path.join(self.pipeline_dir, "subworkflows", "local", "test_subworkflow.nf")) + + +def test_subworkflows_create_fail_exists(self): + """Fail at creating the same subworkflow twice""" + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( + self.pipeline_dir, "test_subworkflow", "@author", False + ) + subworkflow_create.create() + with pytest.raises(UserWarning) as excinfo: + subworkflow_create.create() + assert "Subworkflow file exists already" in str(excinfo.value) + + +def test_subworkflows_create_nfcore_modules(self): + """Create a subworkflow in nf-core/modules clone""" + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( + self.nfcore_modules, "test_subworkflow", "@author", False + ) + subworkflow_create.create() + assert os.path.exists(os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf")) + assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "subworkflows", "test_subworkflow", "main.nf")) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py new file mode 100644 index 0000000000..d4a829c802 --- /dev/null +++ b/tests/test_subworkflows.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +""" Tests covering the subworkflows commands +""" + +import os +import shutil +import tempfile +import unittest + +import nf_core.create +import nf_core.modules +import nf_core.subworkflows + +from .utils import GITLAB_URL + +def create_modules_repo_dummy(tmp_dir): + """Create a dummy copy of the nf-core/modules repo""" + + root_dir = os.path.join(tmp_dir, "modules") + os.makedirs(os.path.join(root_dir, "modules")) + os.makedirs(os.path.join(root_dir,"subworkflows")) + os.makedirs(os.path.join(root_dir,"subworkflows","nf-core")) + os.makedirs(os.path.join(root_dir, "tests", "modules")) + os.makedirs(os.path.join(root_dir, "tests", "subworkflows")) + os.makedirs(os.path.join(root_dir, "tests", "config")) + with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: + fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) + with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + fh.writelines(["repository_type: modules", "\n"]) + + # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) + module_create.create() + + return root_dir + +class TestSubworkflows(unittest.TestCase): + """Class for subworkflows tests""" + + def setUp(self): + """Create a new PipelineStructure and Launch objects""" + self.tmp_dir = tempfile.mkdtemp() + + # Set up the pipeline structure + root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") + self.pipeline_dir = os.path.join(self.tmp_dir, "mypipeline") + nf_core.create.PipelineCreate( + "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True + ).init_pipeline() + + # Set up the nf-core/modules repo dummy + self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) + + ############################################ + # Test of the individual modules commands. # + ############################################ + + from .subworkflows.create import ( + test_subworkflows_create_fail_exists, + test_subworkflows_create_nfcore_modules, + test_subworkflows_create_succeed, + ) + From 72f47cdc768c0d615fdb373e3f140f1b7f57c7b3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 26 Sep 2022 09:50:50 +0200 Subject: [PATCH 129/854] fix `create` help message --- nf_core/__main__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 39cd1390ab..0a0158ee18 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -54,6 +54,12 @@ "commands": ["create", "create-test-yml", "lint", "bump-versions", "mulled", "test"], }, ], + "nf-core subworkflows": [ + { + "name": "Developing new modules", + "commands": ["create"], + }, + ], } click.rich_click.OPTION_GROUPS = { "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}], From 7127d8af1f59ff303451ed8a85cd1d419e681b89 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 26 Sep 2022 11:29:15 +0200 Subject: [PATCH 130/854] allow `_` in subworkflow name --- nf_core/subworkflows/create.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 1a33284a27..fba748fbca 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -83,10 +83,10 @@ def create(self): # Collect module info via prompt if empty or invalid if self.subworkflow is None: self.subworkflow = "" - while self.subworkflow == "" or re.search(r"[^a-z\d/]", self.subworkflow) or self.subworkflow.count("/") > 0: + while self.subworkflow == "" or re.search(r"[^a-z\d_/]", self.subworkflow) or self.subworkflow.count("/") > 0: - # Check + auto-fix for invalid chacters - if re.search(r"[^a-z\d/]", self.subworkflow): + # Check + auto-fix for invalid characters + if re.search(r"[^a-z\d_/]", self.subworkflow): log.warning("Subworkflow name must be lower-case letters only, with no punctuation") subworkflow_clean = re.sub(r"[^a-z\d/]", "", self.subworkflow.lower()) if rich.prompt.Confirm.ask(f"[violet]Change '{self.subworkflow}' to '{subworkflow_clean}'?"): From 4a81e815c58958dab1842dd75099636b2a1b2231 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 26 Sep 2022 11:29:25 +0200 Subject: [PATCH 131/854] fix tests --- nf_core/subworkflows/create.py | 4 +--- tests/subworkflows/create.py | 12 +++++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index fba748fbca..533476ecbe 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -49,8 +49,6 @@ def create(self): e.g bam_sort or bam_sort_samtools, respectively. If is a pipeline, this function creates a file called: - '/subworkflows/local/subworkflow_name.nf' - OR '/subworkflows/local/subworkflow_name.nf' If is a clone of nf-core/modules, it creates or modifies the following files: @@ -192,7 +190,7 @@ def get_subworkflow_dirs(self): raise UserWarning(f"Subworkflow file exists already: '{subworkflow_file}'. Use '--force' to overwrite") # Set file paths - file_paths[os.path.join("subworkflows", f"{self.subworkflow_name}.nf")] = subworkflow_file + file_paths[os.path.join("subworkflows", "main.nf")] = subworkflow_file if self.repo_type == "modules": subworkflow_path = os.path.join(self.directory, "subworkflows", "nf-core", self.subworkflow_dir) diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index 1b838eec1d..ac421064e8 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -7,18 +7,14 @@ def test_subworkflows_create_succeed(self): """Succeed at creating a subworkflow from the template inside a pipeline""" - subworkflow_create = nf_core.subworkflows.SubworkflowCreate( - self.pipeline_dir, "test_subworkflow", "@author", True - ) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(self.pipeline_dir, "test_subworkflow", "@author", True) subworkflow_create.create() assert os.path.exists(os.path.join(self.pipeline_dir, "subworkflows", "local", "test_subworkflow.nf")) def test_subworkflows_create_fail_exists(self): """Fail at creating the same subworkflow twice""" - subworkflow_create = nf_core.subworkflows.SubworkflowCreate( - self.pipeline_dir, "test_subworkflow", "@author", False - ) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(self.pipeline_dir, "test_subworkflow", "@author", False) subworkflow_create.create() with pytest.raises(UserWarning) as excinfo: subworkflow_create.create() @@ -32,4 +28,6 @@ def test_subworkflows_create_nfcore_modules(self): ) subworkflow_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf")) - assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "subworkflows", "test_subworkflow", "main.nf")) + assert os.path.exists( + os.path.join(self.nfcore_modules, "tests", "subworkflows", "nf-core", "test_subworkflow", "main.nf") + ) From d627f3637a9316855071da54894c4238868ba87c Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 26 Sep 2022 11:33:11 +0200 Subject: [PATCH 132/854] fix isort --- tests/test_subworkflows.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index d4a829c802..e5d3f7d3f4 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -13,13 +13,14 @@ from .utils import GITLAB_URL + def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" root_dir = os.path.join(tmp_dir, "modules") os.makedirs(os.path.join(root_dir, "modules")) - os.makedirs(os.path.join(root_dir,"subworkflows")) - os.makedirs(os.path.join(root_dir,"subworkflows","nf-core")) + os.makedirs(os.path.join(root_dir, "subworkflows")) + os.makedirs(os.path.join(root_dir, "subworkflows", "nf-core")) os.makedirs(os.path.join(root_dir, "tests", "modules")) os.makedirs(os.path.join(root_dir, "tests", "subworkflows")) os.makedirs(os.path.join(root_dir, "tests", "config")) @@ -34,6 +35,7 @@ def create_modules_repo_dummy(tmp_dir): return root_dir + class TestSubworkflows(unittest.TestCase): """Class for subworkflows tests""" @@ -61,4 +63,3 @@ def setUp(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) - From 6cfe78e7f8a19a506498ec74aac5e9a367b843ba Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 14:59:22 +0200 Subject: [PATCH 133/854] change path for testing module structure --- nf_core/modules/module_test.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 927fd6b693..ff5ae7d44c 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -114,12 +114,8 @@ def _validate_folder_structure(self): """ basedir = "modules/nf-core" - if self.repo_type == "modules": - module_path = Path("modules") / self.module_name - test_path = Path("tests/modules") / self.module_name - else: - module_path = Path(f"{basedir}/modules") / self.module_name - test_path = Path(f"{basedir}/tests/modules") / self.module_name + module_path = Path(basedir) / self.module_name + test_path = Path(f"tests/{basedir}") / self.module_name if not (self.dir / module_path).is_dir(): raise UserWarning( From ae958d14388173f4e73ae8bf91e26011265b0c00 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 15:02:26 +0200 Subject: [PATCH 134/854] change module folder structure from pipeline template --- .../nf-core/{modules => }/custom/dumpsoftwareversions/main.nf | 0 .../nf-core/{modules => }/custom/dumpsoftwareversions/meta.yml | 0 .../custom/dumpsoftwareversions/templates/dumpsoftwareversions.py | 0 .../modules/nf-core/{modules => }/fastqc/main.nf | 0 .../modules/nf-core/{modules => }/fastqc/meta.yml | 0 .../modules/nf-core/{modules => }/multiqc/main.nf | 0 .../modules/nf-core/{modules => }/multiqc/meta.yml | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename nf_core/pipeline-template/modules/nf-core/{modules => }/custom/dumpsoftwareversions/main.nf (100%) rename nf_core/pipeline-template/modules/nf-core/{modules => }/custom/dumpsoftwareversions/meta.yml (100%) rename nf_core/pipeline-template/modules/nf-core/{modules => }/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py (100%) rename nf_core/pipeline-template/modules/nf-core/{modules => }/fastqc/main.nf (100%) rename nf_core/pipeline-template/modules/nf-core/{modules => }/fastqc/meta.yml (100%) rename nf_core/pipeline-template/modules/nf-core/{modules => }/multiqc/main.nf (100%) rename nf_core/pipeline-template/modules/nf-core/{modules => }/multiqc/meta.yml (100%) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf similarity index 100% rename from nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf rename to nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf diff --git a/nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml similarity index 100% rename from nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml rename to nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml diff --git a/nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py similarity index 100% rename from nf_core/pipeline-template/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py rename to nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf similarity index 100% rename from nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf rename to nf_core/pipeline-template/modules/nf-core/fastqc/main.nf diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml similarity index 100% rename from nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml rename to nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf similarity index 100% rename from nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf rename to nf_core/pipeline-template/modules/nf-core/multiqc/main.nf diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml similarity index 100% rename from nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml rename to nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml From 1c8e6343d8bec0be647a28965be82be81ae879aa Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 15:03:34 +0200 Subject: [PATCH 135/854] change path from pipeline template --- nf_core/pipeline-template/workflows/pipeline.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 83d49a327b..34b3be75b4 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -48,9 +48,9 @@ include { INPUT_CHECK } from '../subworkflows/local/input_check' // // MODULE: Installed directly from nf-core/modules // -include { FASTQC } from '../modules/nf-core/modules/fastqc/main' -include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' +include { FASTQC } from '../modules/nf-core/fastqc/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 11e5d0d94ab58453c32f991e1ea975868d60b9cb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 15:10:11 +0200 Subject: [PATCH 136/854] change path from module_utils --- nf_core/modules/module_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 144f7ce3d4..19f2c7b1b7 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -58,11 +58,11 @@ def get_installed_modules(dir, repo_type="modules"): local_modules = [] nfcore_modules = [] local_modules_dir = None - nfcore_modules_dir = os.path.join(dir, "modules", "nf-core", "modules") + nfcore_modules_dir = os.path.join(dir, "modules", "nf-core") # Get local modules if repo_type == "pipeline": - local_modules_dir = os.path.join(dir, "modules", "local", "process") + local_modules_dir = os.path.join(dir, "modules", "local") # Filter local modules if os.path.exists(local_modules_dir): From ac38a146479d686459dcf9a40ea8ea5b7efc6f58 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 15:22:51 +0200 Subject: [PATCH 137/854] change paths added to pytest_modules.yml --- nf_core/modules/create.py | 12 ++++++------ nf_core/modules/modules_json.py | 2 +- nf_core/modules/modules_repo.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 9ea80cc7c5..af2d80c492 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -72,10 +72,10 @@ def create(self): If is a clone of nf-core/modules, it creates or modifies the following files: - modules/modules/tool/subtool/ + modules/modules/nf-core/tool/subtool/ * main.nf * meta.yml - modules/tests/modules/tool/subtool/ + modules/tests/modules/nf-core/tool/subtool/ * main.nf * test.yml * nextflow.config @@ -250,13 +250,13 @@ def create(self): pytest_modules_yml = yaml.safe_load(fh) if self.subtool: pytest_modules_yml[self.tool_name] = [ - f"modules/{self.tool}/{self.subtool}/**", - f"tests/modules/{self.tool}/{self.subtool}/**", + f"modules/nf-core/{self.tool}/{self.subtool}/**", + f"tests/modules/nf-core/{self.tool}/{self.subtool}/**", ] else: pytest_modules_yml[self.tool_name] = [ - f"modules/{self.tool}/**", - f"tests/modules/{self.tool}/**", + f"modules/nf-core/{self.tool}/**", + f"tests/modules/nf-core/{self.tool}/**", ] pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 005baa84a5..20b2f7095b 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -204,7 +204,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, modules): Returns: (dict[str, dict[str, str]]): The module.json entries for the modules - from the repository + from the repository """ default_modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url) repo_path = self.modules_dir / repo_name diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 1bb2770f33..22be8d3d38 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -362,7 +362,7 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 ( dict ): Iterator of commit SHAs and associated (truncated) message """ self.checkout_branch() - module_path = os.path.join("modules", module_name) + module_path = module_name commits = self.repo.iter_commits(max_count=depth, paths=module_path) commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) return commits From 5235405ad29451887861600bcab4c060f3372dd2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 15:48:56 +0200 Subject: [PATCH 138/854] modify modules create command --- nf_core/modules/create.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index af2d80c492..5b825d1c6c 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -320,11 +320,11 @@ def get_module_dirs(self): ) # Set file paths - file_paths[os.path.join("modules", "main.nf")] = module_file + file_paths[os.path.join("modules", "nf-core", "main.nf")] = module_file if self.repo_type == "modules": - software_dir = os.path.join(self.directory, "modules", self.tool_dir) - test_dir = os.path.join(self.directory, "tests", "modules", self.tool_dir) + software_dir = os.path.join(self.directory, "modules", "nf-core", self.tool_dir) + test_dir = os.path.join(self.directory, "tests", "modules", "nf-core", self.tool_dir) # Check if module directories exist already if os.path.exists(software_dir) and not self.force_overwrite: @@ -334,8 +334,8 @@ def get_module_dirs(self): raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite") # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = os.path.join(self.directory, "modules", self.tool, "main.nf") - parent_tool_test_nf = os.path.join(self.directory, "tests", "modules", self.tool, "main.nf") + parent_tool_main_nf = os.path.join(self.directory, "modules", "nf-core", self.tool, "main.nf") + parent_tool_test_nf = os.path.join(self.directory, "tests", "modules", "nf-core", self.tool, "main.nf") if self.subtool and os.path.exists(parent_tool_main_nf): raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.tool_name}'" @@ -346,7 +346,7 @@ def get_module_dirs(self): ) # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{os.path.join(self.directory, 'modules', self.tool)}/*/main.nf") + tool_glob = glob.glob(f"{os.path.join(self.directory, 'modules', 'nf-core', self.tool)}/*/main.nf") if not self.subtool and tool_glob: raise UserWarning( f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'" From 05f1cb49fc70bc208fe2b5c530ae65e2ce91c1d6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 16:11:51 +0200 Subject: [PATCH 139/854] fix create-test-yml --- nf_core/module-template/tests/main.nf | 2 +- nf_core/modules/test_yml_builder.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/module-template/tests/main.nf b/nf_core/module-template/tests/main.nf index a7be132962..99a042d730 100644 --- a/nf_core/module-template/tests/main.nf +++ b/nf_core/module-template/tests/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { {{ tool_name_underscore|upper }} } from '../../../{{ "../" if subtool else "" }}modules/{{ tool_dir }}/main.nf' +include { {{ tool_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/nf-core/{{ tool_dir }}/main.nf' workflow test_{{ tool_name_underscore }} { {% if has_meta %} diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index a47f7c352a..abfafad597 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -75,8 +75,8 @@ def check_inputs(self): choices=modules_repo.get_avail_modules(), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - self.module_dir = os.path.join("modules", *self.module_name.split("/")) - self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") + self.module_dir = os.path.join("modules", "nf-core", *self.module_name.split("/")) + self.module_test_main = os.path.join("tests", "modules", "nf-core", *self.module_name.split("/"), "main.nf") # First, sanity check that the module directory exists if not os.path.isdir(self.module_dir): @@ -91,7 +91,7 @@ def check_inputs(self): # Get the output YAML file / check it does not already exist while self.test_yml_output_path is None: - default_val = f"tests/modules/{self.module_name}/test.yml" + default_val = f"tests/modules/nf-core/{self.module_name}/test.yml" if self.no_prompts: self.test_yml_output_path = default_val else: @@ -166,7 +166,7 @@ def build_single_test(self, entry_point): while ep_test["command"] == "": # Don't think we need the last `-c` flag, but keeping to avoid having to update 100s modules. # See https://github.com/nf-core/tools/issues/1562 - default_val = f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config -c ./tests/modules/{self.module_name}/nextflow.config" + default_val = f"nextflow run ./tests/modules/nf-core/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/{self.module_name}/nextflow.config" if self.no_prompts: ep_test["command"] = default_val else: From e81615d11ad2bbb356dab3021f98df87569995d8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 26 Sep 2022 16:26:50 +0200 Subject: [PATCH 140/854] Schema: Remove `allOf` if no definition groups are left. --- CHANGELOG.md | 1 + nf_core/schema.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27b86f23ce..ddd36653a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### General - Add function to enable chat notifications on MS Teams, accompanied by `hook_url` param to enable it. +- Schema: Remove `allOf` if no definition groups are left. ### Modules diff --git a/nf_core/schema.py b/nf_core/schema.py index 6804183cb8..df5e691856 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -667,6 +667,15 @@ def remove_schema_empty_definitions(self): if allOf in self.schema.get("allOf", []): self.schema["allOf"].remove(allOf) + # If we don't have anything left in "allOf", remove it + if self.schema.get("allOf") == []: + del self.schema["allOf"] + + # If we don't have anything left in "definitions", remove it + if self.schema.get("definitions") == {}: + del self.schema["definitions"] + + def remove_schema_notfound_configs(self): """ Go through top-level schema and all definitions sub-schemas to remove From 3ce7b8a2f8ee36ac29ade70d2cdcb1b59bfa57ad Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 26 Sep 2022 16:34:25 +0200 Subject: [PATCH 141/854] Don't assume these keys exist any more --- nf_core/schema.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index df5e691856..04e18409cb 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -228,7 +228,7 @@ def validate_default_params(self): schema_no_required = copy.deepcopy(self.schema) if "required" in schema_no_required: schema_no_required.pop("required") - for group_key, group in schema_no_required["definitions"].items(): + for group_key, group in schema_no_required.get("definitions", {}).items(): if "required" in group: schema_no_required["definitions"][group_key].pop("required") jsonschema.validate(self.schema_defaults, schema_no_required) @@ -247,7 +247,7 @@ def validate_default_params(self): params_ignore = [] # Go over group keys - for group_key, group in schema_no_required["definitions"].items(): + for group_key, group in schema_no_required.get("definitions", {}).items(): group_properties = group.get("properties") for param in group_properties: if param in params_ignore: @@ -343,7 +343,7 @@ def validate_schema(self, schema=None): if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") in_allOf = False - for allOf in schema["allOf"]: + for allOf in schema.get("allOf", []): if allOf["$ref"] == f"#/definitions/{d_key}": in_allOf = True if not in_allOf: @@ -361,7 +361,7 @@ def validate_schema(self, schema=None): if "definitions" not in schema: raise AssertionError("Schema has allOf, but no definitions") def_key = allOf["$ref"][14:] - if def_key not in schema["definitions"]: + if def_key not in schema.get("definitions", {}): raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `definitions`") # Check that the schema describes at least one parameter @@ -675,7 +675,6 @@ def remove_schema_empty_definitions(self): if self.schema.get("definitions") == {}: del self.schema["definitions"] - def remove_schema_notfound_configs(self): """ Go through top-level schema and all definitions sub-schemas to remove From 7a2d46eb2b107d265f09afa55dc2a799b04b234e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 26 Sep 2022 16:56:23 +0200 Subject: [PATCH 142/854] add variables to ModuleCommand --- nf_core/modules/create.py | 17 ++++++++++------- nf_core/modules/modules_command.py | 2 ++ nf_core/modules/test_yml_builder.py | 8 +++++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 5b825d1c6c..81f0bc4429 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -22,10 +22,12 @@ import nf_core.modules.module_utils import nf_core.utils +from .modules_command import ModuleCommand + log = logging.getLogger(__name__) -class ModuleCreate(object): +class ModuleCreate(ModuleCommand): def __init__( self, directory=".", @@ -38,6 +40,7 @@ def __init__( conda_version=None, repo_type=None, ): + super().__init__(directory) self.directory = directory self.tool = tool self.author = author @@ -320,11 +323,11 @@ def get_module_dirs(self): ) # Set file paths - file_paths[os.path.join("modules", "nf-core", "main.nf")] = module_file + file_paths[os.path.join(self.default_modules_path, "main.nf")] = module_file if self.repo_type == "modules": - software_dir = os.path.join(self.directory, "modules", "nf-core", self.tool_dir) - test_dir = os.path.join(self.directory, "tests", "modules", "nf-core", self.tool_dir) + software_dir = os.path.join(self.directory, self.default_modules_path, self.tool_dir) + test_dir = os.path.join(self.directory, self.default_tests_path, self.tool_dir) # Check if module directories exist already if os.path.exists(software_dir) and not self.force_overwrite: @@ -334,8 +337,8 @@ def get_module_dirs(self): raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite") # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = os.path.join(self.directory, "modules", "nf-core", self.tool, "main.nf") - parent_tool_test_nf = os.path.join(self.directory, "tests", "modules", "nf-core", self.tool, "main.nf") + parent_tool_main_nf = os.path.join(self.directory, self.default_modules_path, self.tool, "main.nf") + parent_tool_test_nf = os.path.join(self.directory, self.default_tests_path, self.tool, "main.nf") if self.subtool and os.path.exists(parent_tool_main_nf): raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.tool_name}'" @@ -346,7 +349,7 @@ def get_module_dirs(self): ) # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{os.path.join(self.directory, 'modules', 'nf-core', self.tool)}/*/main.nf") + tool_glob = glob.glob(f"{os.path.join(self.directory, self.default_modules_path, self.tool)}/*/main.nf") if not self.subtool and tool_glob: raise UserWarning( f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'" diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 7a24183271..1bd673a823 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -26,6 +26,8 @@ def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progre self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress = hide_progress self.dir = dir + self.default_modules_path = Path("modules", "nf-core") + self.default_tests_path = Path("tests", "modules", "nf-core") try: if self.dir: self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index abfafad597..7cbdd94b3b 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -25,12 +25,13 @@ import nf_core.utils +from .modules_command import ModuleCommand from .modules_repo import ModulesRepo log = logging.getLogger(__name__) -class ModulesTestYmlBuilder(object): +class ModulesTestYmlBuilder(ModuleCommand): def __init__( self, module_name=None, @@ -39,6 +40,7 @@ def __init__( force_overwrite=False, no_prompts=False, ): + super().__init__(".") self.module_name = module_name self.run_tests = run_tests self.test_yml_output_path = test_yml_output_path @@ -75,8 +77,8 @@ def check_inputs(self): choices=modules_repo.get_avail_modules(), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - self.module_dir = os.path.join("modules", "nf-core", *self.module_name.split("/")) - self.module_test_main = os.path.join("tests", "modules", "nf-core", *self.module_name.split("/"), "main.nf") + self.module_dir = os.path.join(self.default_modules_path, *self.module_name.split("/")) + self.module_test_main = os.path.join(self.default_tests_path, *self.module_name.split("/"), "main.nf") # First, sanity check that the module directory exists if not os.path.isdir(self.module_dir): From e2ec9a7493474e0f803cb2fe4d8bfe7a1da0dfeb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Sep 2022 09:26:54 +0200 Subject: [PATCH 143/854] fix paths for linting modules --- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/modules_command.py | 2 +- nf_core/modules/nfcore_module.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index dd1fd5b7d2..f4db945cc4 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -102,7 +102,7 @@ def __init__( else: raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.") else: - module_dir = Path(self.dir, "modules") + module_dir = Path(self.dir, self.default_modules_path) self.all_remote_modules = [ NFCoreModule(m, None, module_dir / m, self.repo_type, Path(self.dir)) for m in self.get_modules_clone_modules() diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 1bd673a823..9dfffa6e9b 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -40,7 +40,7 @@ def get_modules_clone_modules(self): """ Get the modules available in a clone of nf-core/modules """ - module_base_path = Path(self.dir, "modules") + module_base_path = Path(self.dir, self.default_modules_path) return [ str(Path(dir).relative_to(module_base_path)) for dir, _, files in os.walk(module_base_path) diff --git a/nf_core/modules/nfcore_module.py b/nf_core/modules/nfcore_module.py index 2654a4ebbb..62b77ab35d 100644 --- a/nf_core/modules/nfcore_module.py +++ b/nf_core/modules/nfcore_module.py @@ -43,7 +43,7 @@ def __init__(self, module_name, repo_name, module_dir, repo_type, base_dir, nf_c self.main_nf = self.module_dir / "main.nf" self.meta_yml = self.module_dir / "meta.yml" - self.test_dir = Path(self.base_dir, "tests", "modules", self.module_name) + self.test_dir = Path(self.base_dir, "tests", self.module_dir) self.test_yml = self.test_dir / "test.yml" self.test_main_nf = self.test_dir / "main.nf" From a7d62a3014e3a57a65932761edeb8ef052de4ee6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Sep 2022 09:45:35 +0200 Subject: [PATCH 144/854] fix paths for bump-versions --- nf_core/modules/bump_versions.py | 1 + nf_core/modules/module_utils.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index a30f93eda6..3e318db418 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -8,6 +8,7 @@ import logging import re +from pathlib import Path import questionary import rich diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 19f2c7b1b7..3bfa20b6a8 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -69,10 +69,6 @@ def get_installed_modules(dir, repo_type="modules"): local_modules = os.listdir(local_modules_dir) local_modules = sorted([x for x in local_modules if x.endswith(".nf")]) - # nf-core/modules - if repo_type == "modules": - nfcore_modules_dir = os.path.join(dir, "modules") - # Get nf-core modules if os.path.exists(nfcore_modules_dir): for m in sorted([m for m in os.listdir(nfcore_modules_dir) if not m == "lib"]): From 28efe298f3e5b3d5b6c9f9318132b0957789c199 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 27 Sep 2022 11:50:41 +0200 Subject: [PATCH 145/854] More helpful error messages if `nf-core download` can't parse a singularity image download --- CHANGELOG.md | 1 + nf_core/download.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4e55e2172..dc299a0cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Add function to enable chat notifications on MS Teams, accompanied by `hook_url` param to enable it. - Schema: Remove `allOf` if no definition groups are left. - Use contextlib to temporarily change working directories ([#1819](https://github.com/nf-core/tools/pull/1819)) +- More helpful error messages if `nf-core download` can't parse a singularity image download ### Modules diff --git a/nf_core/download.py b/nf_core/download.py index e9a193b2a0..923d953fae 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -12,6 +12,7 @@ import subprocess import sys import tarfile +import textwrap from zipfile import ZipFile import questionary @@ -453,7 +454,8 @@ def find_container_images(self): for subdir, _, files in os.walk(os.path.join(self.outdir, "workflow", "modules")): for file in files: if file.endswith(".nf"): - with open(os.path.join(subdir, file), "r") as fh: + file_path = os.path.join(subdir, file) + with open(file_path, "r") as fh: # Look for any lines with `container = "xxx"` this_container = None contents = fh.read() @@ -478,7 +480,9 @@ def find_container_images(self): # Don't recognise this, throw a warning else: - log.error(f"[red]Cannot parse container string, skipping: [green]'{file}'") + log.error( + f"[red]Cannot parse container string in '{file_path}':\n\n{textwrap.indent(match, ' ')}\n\n:warning: Skipping this singularity image.." + ) if this_container: containers_raw.append(this_container) From 3cd133608ac4f7129ca66a03044cd28559458db4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Sep 2022 15:48:56 +0200 Subject: [PATCH 146/854] try to add any custom folder as nf-core inside modules --- nf_core/modules/module_test.py | 10 ++++------ nf_core/modules/modules_json.py | 20 +++++++++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index ff5ae7d44c..339f5a1eb7 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -108,14 +108,12 @@ def _check_inputs(self): def _validate_folder_structure(self): """Validate that the modules follow the correct folder structure to run the tests: - - modules/TOOL/SUBTOOL/ - - tests/modules/TOOL/SUBTOOL/ + - modules/nf-core/TOOL/SUBTOOL/ + - tests/modules/nf-core/TOOL/SUBTOOL/ """ - basedir = "modules/nf-core" - - module_path = Path(basedir) / self.module_name - test_path = Path(f"tests/{basedir}") / self.module_name + module_path = Path(self.default_modules_path) / self.module_name + test_path = Path(self.default_tests_path) / self.module_name if not (self.dir / module_path).is_dir(): raise UserWarning( diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 20b2f7095b..caa069e3d2 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -60,7 +60,12 @@ def create(self): ( repo_name, [ - str(Path(dir_name).relative_to(modules_dir / repo_name)) + str(Path(dir_name).relative_to(modules_dir / repo_name).parts[0]) + for dir_name, _, file_names in os.walk(modules_dir / repo_name) + if "main.nf" in file_names + ], + [ + str(Path(dir_name).relative_to(modules_dir / repo_name).parts[1:]) for dir_name, _, file_names in os.walk(modules_dir / repo_name) if "main.nf" in file_names ], @@ -69,12 +74,12 @@ def create(self): for repo_name, repo_remote in repos.items() ] - for repo_name, module_names, remote_url in sorted(repo_module_names): + for repo_name, dir_names, module_names, remote_url in sorted(repo_module_names): modules_json["repos"][repo_name] = {} modules_json["repos"][repo_name]["git_url"] = remote_url - modules_json["repos"][repo_name]["modules"] = {} - modules_json["repos"][repo_name]["modules"] = self.determine_module_branches_and_shas( - repo_name, remote_url, module_names + modules_json["repos"][repo_name]["modules"][dir_names] = {} + modules_json["repos"][repo_name]["modules"][dir_names] = self.determine_module_branches_and_shas( + Path(repo_name / dir_names), remote_url, module_names ) # write the modules.json file and assign it to the object modules_json_path = Path(self.dir, "modules.json") @@ -204,7 +209,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, modules): Returns: (dict[str, dict[str, str]]): The module.json entries for the modules - from the repository + from the repository """ default_modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url) repo_path = self.modules_dir / repo_name @@ -447,7 +452,8 @@ def check_up_to_date(self): missing_but_in_mod_json = [ f"'{repo}/{module}'" for repo, contents in missing_installation.items() - for module in contents["modules"] + for module_dir in contents["modules"].keys() + for module in contents["modules"][module_dir] ] log.info( f"Reinstalling modules found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" From 139818fcae18a2ab67bca7deb492eda5808f02cc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Sep 2022 16:53:31 +0200 Subject: [PATCH 147/854] modules_json.py working (hopefully) --- nf_core/modules/modules_json.py | 137 +++++++++++++++----------------- 1 file changed, 64 insertions(+), 73 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index caa069e3d2..a6d1fc4afc 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -33,7 +33,7 @@ def __init__(self, pipeline_dir): pipeline_dir (str): The pipeline directory """ self.dir = pipeline_dir - self.modules_dir = Path(self.dir, "modules") + self.modules_dir = Path(self.dir, "modules", "nf-core") self.modules_json = None self.pipeline_modules = None @@ -48,7 +48,7 @@ def create(self): pipeline_name = pipeline_config.get("manifest.name", "") pipeline_url = pipeline_config.get("manifest.homePage", "") modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": {}} - modules_dir = Path(self.dir, "modules") + modules_dir = Path(self.dir, "modules/nf-core") if not modules_dir.exists(): raise UserWarning("Can't find a ./modules directory. Is this a DSL2 pipeline?") @@ -60,12 +60,7 @@ def create(self): ( repo_name, [ - str(Path(dir_name).relative_to(modules_dir / repo_name).parts[0]) - for dir_name, _, file_names in os.walk(modules_dir / repo_name) - if "main.nf" in file_names - ], - [ - str(Path(dir_name).relative_to(modules_dir / repo_name).parts[1:]) + str(Path(dir_name).relative_to(modules_dir / repo_name)) for dir_name, _, file_names in os.walk(modules_dir / repo_name) if "main.nf" in file_names ], @@ -74,12 +69,11 @@ def create(self): for repo_name, repo_remote in repos.items() ] - for repo_name, dir_names, module_names, remote_url in sorted(repo_module_names): - modules_json["repos"][repo_name] = {} - modules_json["repos"][repo_name]["git_url"] = remote_url - modules_json["repos"][repo_name]["modules"][dir_names] = {} - modules_json["repos"][repo_name]["modules"][dir_names] = self.determine_module_branches_and_shas( - Path(repo_name / dir_names), remote_url, module_names + for repo_name, module_names, remote_url in sorted(repo_module_names): + modules_json["repos"][remote_url] = {} + modules_json["repos"][remote_url]["modules/nf-core"] = {} + modules_json["repos"][remote_url]["modules/nf-core"] = self.determine_module_branches_and_shas( + repo_name, remote_url, module_names ) # write the modules.json file and assign it to the object modules_json_path = Path(self.dir, "modules.json") @@ -156,7 +150,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): else: continue - repos[nrepo_name] = (nrepo_remote, "modules") + repos[nrepo_name] = (nrepo_remote, "modules/nf-core") dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos]) return repos, renamed_dirs @@ -337,29 +331,30 @@ def unsynced_modules(self): ] untracked_dirs = [] for dir in dirs: + print(f"check if {dir} is installed") # Check if the modules directory exists module_repo_name = None + git_url = None for repo in missing_installation: - if str(dir).startswith(repo + os.sep): - module_repo_name = repo + for dir_name in missing_installation[repo]: + module_repo_name = dir_name + git_url = repo break if module_repo_name is not None: # If it does, check if the module is in the 'modules.json' file - module = str(dir.relative_to(module_repo_name)) - module_repo = missing_installation[module_repo_name] + module = str(dir) + module_repo = missing_installation[git_url] - if module not in module_repo.get("modules", {}): + if module not in module_repo["modules/nf-core"].keys(): untracked_dirs.append(dir) else: # Check if the entry has a git sha and branch before removing - modules = module_repo["modules"] + modules = module_repo["modules/nf-core"] if "git_sha" not in modules[module] or "branch" not in modules[module]: - self.determine_module_branches_and_shas( - module, module_repo["git_url"], module_repo["base_path"], [module] - ) - module_repo["modules"].pop(module) - if len(module_repo["modules"]) == 0: - missing_installation.pop(module_repo_name) + self.determine_module_branches_and_shas(module, git_url, module_repo["base_path"], [module]) + module_repo["modules/nf-core"].pop(module) + if len(module_repo["modules/nf-core"]) == 0: + missing_installation.pop(git_url) else: # If it is not, add it to the list of missing modules untracked_dirs.append(dir) @@ -373,15 +368,15 @@ def has_git_url_and_modules(self): Returns: (bool): True if they are found for all repos, False otherwise """ - for repo_entry in self.modules_json.get("repos", {}).values(): - if "git_url" not in repo_entry or "modules" not in repo_entry: - log.warning(f"modules.json entry {repo_entry} does not have a git_url or modules entry") + for repo_url, repo_entry in self.modules_json.get("repos", {}).items(): + if "modules/nf-core" not in repo_entry: + log.warning(f"modules.json entry {repo_entry} does not have a modules entry") return False elif ( - not isinstance(repo_entry["git_url"], str) - or repo_entry["git_url"] == "" - or not isinstance(repo_entry["modules"], dict) - or repo_entry["modules"] == {} + not isinstance(repo_url, str) + or repo_url == "" + or not isinstance(repo_entry["modules/nf-core"], dict) + or repo_entry["modules/nf-core"] == {} ): log.warning(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules") return False @@ -419,7 +414,7 @@ def reinstall_repo(self, repo_name, remote_url, module_entries): log.error(e) failed_to_install.extend(modules) for module, sha in modules: - if not modules_repo.install_module(module, (self.modules_dir / repo_name), sha): + if not modules_repo.install_module(module, self.modules_dir, sha): log.warning(f"Could not install module '{Path(repo_name, module)}' - removing from modules.json") failed_to_install.append(module) return failed_to_install @@ -450,10 +445,9 @@ def check_up_to_date(self): # we try to reinstall them if len(missing_installation) > 0: missing_but_in_mod_json = [ - f"'{repo}/{module}'" - for repo, contents in missing_installation.items() - for module_dir in contents["modules"].keys() - for module in contents["modules"][module_dir] + f"'modules/nf-core/{module}'" + for repo_url, contents in missing_installation.items() + for module in contents["modules/nf-core"] ] log.info( f"Reinstalling modules found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" @@ -461,8 +455,8 @@ def check_up_to_date(self): remove_from_mod_json = {} for repo, contents in missing_installation.items(): - module_entries = contents["modules"] - remote_url = contents["git_url"] + module_entries = contents["modules/nf-core"] + remote_url = repo remove_from_mod_json[repo] = self.reinstall_repo(repo, remote_url, module_entries) # If the reinstall fails, we remove those entries in 'modules.json' @@ -479,8 +473,8 @@ def check_up_to_date(self): for repo, module_entries in remove_from_mod_json.items(): for module in module_entries: - self.modules_json["repos"][repo]["modules"].pop(module) - if len(self.modules_json["repos"][repo]["modules"]) == 0: + self.modules_json["repos"][repo]["modules/nf-core"].pop(module) + if len(self.modules_json["repos"][repo]["modules/nf-core"]) == 0: self.modules_json["repos"].pop(repo) # If some modules didn't have an entry in the 'modules.json' file @@ -495,9 +489,7 @@ def check_up_to_date(self): ) # Get the remotes we are missing - tracked_repos = { - repo_name: (repo_entry["git_url"]) for repo_name, repo_entry in self.modules_json["repos"].items() - } + tracked_repos = {repo_name: (remote_url) for repo_name, repo_entry in self.modules_json["repos"].items()} repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) modules_with_repos = ( @@ -517,11 +509,10 @@ def check_up_to_date(self): remote_url = repos[repo_name] repo_entry = self.determine_module_branches_and_shas(repo_name, remote_url, modules) if repo_name in self.modules_json["repos"]: - self.modules_json["repos"][repo_name]["modules"].update(repo_entry) + self.modules_json["repos"][repo_name]["modules/nf-core"].update(repo_entry) else: self.modules_json["repos"][repo_name] = { - "git_url": remote_url, - "modules": repo_entry, + "modules/nf-core": repo_entry, } self.dump() @@ -558,8 +549,8 @@ def update(self, modules_repo, module_name, module_version, write_file=True): remote_url = modules_repo.remote_url branch = modules_repo.branch if repo_name not in self.modules_json["repos"]: - self.modules_json["repos"][repo_name] = {"modules": {}, "git_url": remote_url} - repo_modules_entry = self.modules_json["repos"][repo_name]["modules"] + self.modules_json["repos"][repo_name] = {"modules/nf-core": {}} + repo_modules_entry = self.modules_json["repos"][repo_name]["modules/nf-core"] if module_name not in repo_modules_entry: repo_modules_entry[module_name] = {} repo_modules_entry[module_name]["git_sha"] = module_version @@ -584,12 +575,12 @@ def remove_entry(self, module_name, repo_name): return False if repo_name in self.modules_json.get("repos", {}): repo_entry = self.modules_json["repos"][repo_name] - if module_name in repo_entry.get("modules", {}): - repo_entry["modules"].pop(module_name) + if module_name in repo_entry.get("modules/nf-core", {}): + repo_entry["modules/nf-core"].pop(module_name) else: log.warning(f"Module '{repo_name}/{module_name}' is missing from 'modules.json' file.") return False - if len(repo_entry["modules"]) == 0: + if len(repo_entry["modules/nf-core"]) == 0: self.modules_json["repos"].pop(repo_name) else: log.warning(f"Module '{repo_name}/{module_name}' is missing from 'modules.json' file.") @@ -606,9 +597,9 @@ def add_patch_entry(self, module_name, repo_name, patch_filename, write_file=Tru self.load() if repo_name not in self.modules_json["repos"]: raise LookupError(f"Repo '{repo_name}' not present in 'modules.json'") - if module_name not in self.modules_json["repos"][repo_name]["modules"]: + if module_name not in self.modules_json["repos"][repo_name]["modules/nf-core"]: raise LookupError(f"Module '{repo_name}/{module_name}' not present in 'modules.json'") - self.modules_json["repos"][repo_name]["modules"][module_name]["patch"] = str(patch_filename) + self.modules_json["repos"][repo_name]["modules/nf-core"][module_name]["patch"] = str(patch_filename) if write_file: self.dump() @@ -625,7 +616,7 @@ def get_patch_fn(self, module_name, repo_name): """ if self.modules_json is None: self.load() - path = self.modules_json["repos"].get(repo_name, {}).get("modules").get(module_name, {}).get("patch") + path = self.modules_json["repos"].get(repo_name, {}).get("modules/nf-core").get(module_name, {}).get("patch") return Path(path) if path is not None else None def try_apply_patch_reverse(self, module, repo_name, patch_relpath, module_dir): @@ -687,7 +678,7 @@ def module_present(self, module_name, repo_name): """ if self.modules_json is None: self.load() - return module_name in self.modules_json.get("repos", {}).get(repo_name, {}).get("modules", {}) + return module_name in self.modules_json.get("repos", {}).get(repo_name, {}).get("modules/nf-core", {}) def get_modules_json(self): """ @@ -716,24 +707,24 @@ def get_module_version(self, module_name, repo_name): return ( self.modules_json.get("repos", {}) .get(repo_name, {}) - .get("modules", {}) + .get("modules/nf-core", {}) .get(module_name, {}) .get("git_sha", None) ) - def get_git_url(self, repo_name): - """ - Returns the git url of a repo + # def get_git_url(self, repo_name): + # """ + # Returns the git url of a repo - Args: - repo_name (str): Name of the repository + # Args: + # repo_name (str): Name of the repository - Returns: - (str): The git url of the repository if it exists, None otherwise - """ - if self.modules_json is None: - self.load() - return self.modules_json.get("repos", {}).get(repo_name, {}).get("git_url", None) + # Returns: + # (str): The git url of the repository if it exists, None otherwise + # """ + # if self.modules_json is None: + # self.load() + # return self.modules_json.get("repos", {}).get(repo_name, {}).get("git_url", None) def get_all_modules(self): """ @@ -748,8 +739,8 @@ def get_all_modules(self): if self.pipeline_modules is None: self.pipeline_modules = {} for repo, repo_entry in self.modules_json.get("repos", {}).items(): - if "modules" in repo_entry: - self.pipeline_modules[repo] = list(repo_entry["modules"]) + if "modules/nf-core" in repo_entry: + self.pipeline_modules[repo] = list(repo_entry["modules/nf-core"]) return self.pipeline_modules @@ -764,7 +755,7 @@ def get_module_branch(self, module, repo_name): """ if self.modules_json is None: self.load() - branch = self.modules_json["repos"].get(repo_name, {}).get("modules", {}).get(module, {}).get("branch") + branch = self.modules_json["repos"].get(repo_name, {}).get("modules/nf-core", {}).get(module, {}).get("branch") if branch is None: raise LookupError( f"Could not find branch information for module '{Path(repo_name, module)}'." From 2d40c7ef005b462a52548440d811dacf9556a252 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Sep 2022 17:09:37 +0200 Subject: [PATCH 148/854] lint bug fixed but lint not passing :( --- nf_core/modules/lint/__init__.py | 6 +++--- nf_core/modules/modules_json.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index f4db945cc4..c4d439812e 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -85,11 +85,11 @@ def __init__( modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() all_pipeline_modules = modules_json.get_all_modules() - if self.modules_repo.fullname in all_pipeline_modules: - module_dir = Path(self.dir, "modules", self.modules_repo.fullname) + if self.modules_repo.remote_url in all_pipeline_modules: + module_dir = Path(self.dir, "modules", "nf-core") self.all_remote_modules = [ NFCoreModule(m, self.modules_repo.fullname, module_dir / m, self.repo_type, Path(self.dir)) - for m in all_pipeline_modules[self.modules_repo.fullname] + for m in all_pipeline_modules[self.modules_repo.remote_url] ] if not self.all_remote_modules: raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.") diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index a6d1fc4afc..2f4f36aa92 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -331,7 +331,6 @@ def unsynced_modules(self): ] untracked_dirs = [] for dir in dirs: - print(f"check if {dir} is installed") # Check if the modules directory exists module_repo_name = None git_url = None From d2840e9a20fd85460413cfdff4d28e3d0b29c77d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 27 Sep 2022 19:54:20 +0200 Subject: [PATCH 149/854] Check that the old renamed `lib` files are not still present See also: nf-core/methylseq#246 --- CHANGELOG.md | 5 +++++ nf_core/lint/files_exist.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc299a0cb1..f6538a3136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ ### Linting +- Pipelines: Check that the old renamed `lib` files are not still present: + - `Checks.groovy` -> `Utils.groovy` + - `Completion.groovy` -> `NfcoreTemplate.groovy` + - `Workflow.groovy` -> `WorkflowMain.groovy` + ### General - Add function to enable chat notifications on MS Teams, accompanied by `hook_url` param to enable it. diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index bc0f732b2e..2a1dee62c9 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -75,7 +75,7 @@ def files_exist(self): lib/WorkflowPIPELINE.groovy pyproject.toml - Files that *must not* be present: + Files that *must not* be present, due to being renamed or removed in the template: .. code-block:: bash @@ -90,6 +90,9 @@ def files_exist(self): docs/images/nf-core-PIPELINE_logo.png .markdownlint.yml .yamllint.yml + lib/Checks.groovy + lib/Completion.groovy + lib/Workflow.groovy Files that *should not* be present: @@ -191,6 +194,9 @@ def files_exist(self): os.path.join("docs", "images", f"nf-core-{short_name}_logo.png"), ".markdownlint.yml", ".yamllint.yml", + os.path.join("lib", "Checks.groovy"), + os.path.join("lib", "Completion.groovy"), + os.path.join("lib", "Workflow.groovy"), ] files_warn_ifexists = [".travis.yml"] From 1be5d6bec4c61ff195fe316aba37fb64d9e09138 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 29 Sep 2022 13:58:17 +0200 Subject: [PATCH 150/854] Modules: If something is wrong with the local repo cache, offer to delete it and try again Closes nf-core/tools#1850 --- CHANGELOG.md | 1 + nf_core/modules/modules_repo.py | 106 +++++++++++++++++--------------- 2 files changed, 59 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c6bd3609a..1b1c0f7854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Schema: Remove `allOf` if no definition groups are left. - Use contextlib to temporarily change working directories ([#1819](https://github.com/nf-core/tools/pull/1819)) - More helpful error messages if `nf-core download` can't parse a singularity image download +- Modules: If something is wrong with the local repo cache, offer to delete it and try again ([#1850](https://github.com/nf-core/tools/issues/1850)) ### Modules diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 1bb2770f33..0a91229aab 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -5,8 +5,9 @@ from pathlib import Path import git +import rich import rich.progress -from git.exc import GitCommandError +from git.exc import GitCommandError, InvalidGitRepositoryError import nf_core.modules.module_utils import nf_core.modules.modules_json @@ -150,55 +151,64 @@ def setup_local_repo(self, remote, branch, hide_progress=True): Sets self.repo """ self.local_repo_dir = os.path.join(NFCORE_DIR, self.fullname) - if not os.path.exists(self.local_repo_dir): - try: - pbar = rich.progress.Progress( - "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), - "[bold yellow]{task.fields[state]}", - transient=True, - disable=hide_progress, - ) - with pbar: - self.repo = git.Repo.clone_from( - remote, - self.local_repo_dir, - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), + try: + if not os.path.exists(self.local_repo_dir): + try: + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress, ) - ModulesRepo.update_local_repo_status(self.fullname, True) - except GitCommandError: - raise LookupError(f"Failed to clone from the remote: `{remote}`") - # Verify that the requested branch exists by checking it out - self.setup_branch(branch) - else: - self.repo = git.Repo(self.local_repo_dir) - - if ModulesRepo.no_pull_global: - ModulesRepo.update_local_repo_status(self.fullname, True) - # If the repo is already cloned, fetch the latest changes from the remote - if not ModulesRepo.local_repo_synced(self.fullname): - pbar = rich.progress.Progress( - "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), - "[bold yellow]{task.fields[state]}", - transient=True, - disable=hide_progress, - ) - with pbar: - self.repo.remotes.origin.fetch( - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + with pbar: + self.repo = git.Repo.clone_from( + remote, + self.local_repo_dir, + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"), + ) + ModulesRepo.update_local_repo_status(self.fullname, True) + except GitCommandError: + raise LookupError(f"Failed to clone from the remote: `{remote}`") + # Verify that the requested branch exists by checking it out + self.setup_branch(branch) + else: + self.repo = git.Repo(self.local_repo_dir) + + if ModulesRepo.no_pull_global: + ModulesRepo.update_local_repo_status(self.fullname, True) + # If the repo is already cloned, fetch the latest changes from the remote + if not ModulesRepo.local_repo_synced(self.fullname): + pbar = rich.progress.Progress( + "[bold blue]{task.description}", + rich.progress.BarColumn(bar_width=None), + "[bold yellow]{task.fields[state]}", + transient=True, + disable=hide_progress, ) - ModulesRepo.update_local_repo_status(self.fullname, True) - - # Before verifying the branch, fetch the changes - # Verify that the requested branch exists by checking it out - self.setup_branch(branch) - - # Now merge the changes - tracking_branch = self.repo.active_branch.tracking_branch() - if tracking_branch is None: - raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{self.remote_url}'") - self.repo.git.merge(tracking_branch.name) + with pbar: + self.repo.remotes.origin.fetch( + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + ) + ModulesRepo.update_local_repo_status(self.fullname, True) + + # Before verifying the branch, fetch the changes + # Verify that the requested branch exists by checking it out + self.setup_branch(branch) + + # Now merge the changes + tracking_branch = self.repo.active_branch.tracking_branch() + if tracking_branch is None: + raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{self.remote_url}'") + self.repo.git.merge(tracking_branch.name) + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n") + if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"): + log.info(f"Removing '{self.local_repo_dir}'") + shutil.rmtree(self.local_repo_dir) + self.setup_local_repo(remote, branch, hide_progress) + else: + raise LookupError("Exiting due to error with local modules git repo") def setup_branch(self, branch): """ From cb7e95ad8ba4d4ac713aa88a4b54148f2f474318 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Sep 2022 09:54:48 +0200 Subject: [PATCH 151/854] pipeline linting passing --- nf_core/lint/modules_json.py | 16 ++++++++-------- nf_core/modules/install.py | 2 +- nf_core/modules/lint/module_version.py | 2 +- nf_core/modules/modules_json.py | 12 ++++++------ nf_core/modules/modules_repo.py | 2 +- nf_core/modules/patch.py | 4 ++-- nf_core/modules/update.py | 4 ++-- nf_core/pipeline-template/modules.json | 5 ++--- tests/modules/install.py | 2 +- tests/modules/modules_json.py | 6 +++--- tests/modules/update.py | 14 +++++++------- 11 files changed, 34 insertions(+), 35 deletions(-) diff --git a/nf_core/lint/modules_json.py b/nf_core/lint/modules_json.py index 8b3e00b945..475113fa00 100644 --- a/nf_core/lint/modules_json.py +++ b/nf_core/lint/modules_json.py @@ -21,31 +21,31 @@ def modules_json(self): _modules_json = ModulesJson(self.wf_path) _modules_json.load() modules_json_dict = _modules_json.modules_json - modules_dir = Path(self.wf_path, "modules") + modules_dir = Path(self.wf_path, "modules", "nf-core") if _modules_json: all_modules_passed = True for repo in modules_json_dict["repos"].keys(): # Check if the modules.json has been updated to keep the - if "modules" not in modules_json_dict["repos"][repo] or "git_url" not in modules_json_dict["repos"][repo]: + if "modules/nf-core" not in modules_json_dict["repos"][repo] or not repo.startswith("http"): failed.append( "Your `modules.json` file is outdated. " - "Please remove it and reinstall it by running any module command." + "It will be automatically generated by running any module command." ) continue - for module, module_entry in modules_json_dict["repos"][repo]["modules"].items(): - if not Path(modules_dir, repo, module).exists(): + for module, module_entry in modules_json_dict["repos"][repo]["modules/nf-core"].items(): + if not Path(modules_dir, module).exists(): failed.append( - f"Entry for `{Path(repo, module)}` found in `modules.json` but module is not installed in " + f"Entry for `{Path(modules_dir, module)}` found in `modules.json` but module is not installed in " "pipeline." ) all_modules_passed = False if module_entry.get("branch") is None: - failed.append(f"Entry for `{Path(repo, module)}` is missing branch information.") + failed.append(f"Entry for `{Path(modules_dir, module)}` is missing branch information.") if module_entry.get("git_sha") is None: - failed.append(f"Entry for `{Path(repo, module)}` is missing version information.") + failed.append(f"Entry for `{Path(modules_dir, module)}` is missing version information.") if all_modules_passed: passed.append("Only installed modules found in `modules.json`") else: diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 926d8e93a6..07cb01d406 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -71,7 +71,7 @@ def install(self, module): log.warning(warn_msg) return False - current_version = modules_json.get_module_version(module, self.modules_repo.fullname) + current_version = modules_json.get_module_version(module, self.modules_repo.remote_url) # Set the install folder based on the repository name install_folder = os.path.join(self.dir, "modules", self.modules_repo.fullname) diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index 2312560365..8f179ba8db 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -26,7 +26,7 @@ def module_version(module_lint_object, module): # Verify that a git_sha exists in the `modules.json` file for this module version = module_lint_object.modules_json.get_module_version( - module.module_name, module_lint_object.modules_repo.fullname + module.module_name, module_lint_object.modules_repo.remote_url ) if version is None: module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path)) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 2f4f36aa92..904d74e5a7 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -690,13 +690,13 @@ def get_modules_json(self): self.load() return copy.deepcopy(self.modules_json) - def get_module_version(self, module_name, repo_name): + def get_module_version(self, module_name, repo_url): """ Returns the version of a module Args: module_name (str): Name of the module - repo_name (str): Name of the repository + repo_url (str): URL of the repository Returns: (str): The git SHA of the module if it exists, None otherwise @@ -705,7 +705,7 @@ def get_module_version(self, module_name, repo_name): self.load() return ( self.modules_json.get("repos", {}) - .get(repo_name, {}) + .get(repo_url, {}) .get("modules/nf-core", {}) .get(module_name, {}) .get("git_sha", None) @@ -743,7 +743,7 @@ def get_all_modules(self): return self.pipeline_modules - def get_module_branch(self, module, repo_name): + def get_module_branch(self, module, repo_url): """ Gets the branch from which the module was installed @@ -754,10 +754,10 @@ def get_module_branch(self, module, repo_name): """ if self.modules_json is None: self.load() - branch = self.modules_json["repos"].get(repo_name, {}).get("modules/nf-core", {}).get(module, {}).get("branch") + branch = self.modules_json["repos"].get(repo_url, {}).get("modules/nf-core", {}).get(module, {}).get("branch") if branch is None: raise LookupError( - f"Could not find branch information for module '{Path(repo_name, module)}'." + f"Could not find branch information for module '{Path('modules/nf-core', module)}'." f"Please remove the 'modules.json' and rerun the command to recreate it" ) return branch diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 22be8d3d38..1bb2770f33 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -362,7 +362,7 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 ( dict ): Iterator of commit SHAs and associated (truncated) message """ self.checkout_branch() - module_path = module_name + module_path = os.path.join("modules", module_name) commits = self.repo.iter_commits(max_count=depth, paths=module_path) commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) return commits diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index b907256bcf..f99c91913a 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -46,13 +46,13 @@ def patch(self, module=None): f"The '{module_fullname}' module does not have an entry in the 'modules.json' file. Cannot compute patch" ) - module_version = self.modules_json.get_module_version(module, self.modules_repo.fullname) + module_version = self.modules_json.get_module_version(module, self.modules_repo.remote_url) if module_version is None: raise UserWarning( f"The '{module_fullname}' module does not have a valid version in the 'modules.json' file. Cannot compute patch" ) # Get the module branch and reset it in the ModulesRepo object - module_branch = self.modules_json.get_module_branch(module, self.modules_repo.fullname) + module_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url) if module_branch != self.modules_repo.branch: self.modules_repo.setup_branch(module_branch) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 124baccfb5..cdcbe35b88 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -134,7 +134,7 @@ def update(self, module=None): # Are we updating the files in place or not? dry_run = self.show_diff or self.save_diff_fn - current_version = self.modules_json.get_module_version(module, modules_repo.fullname) + current_version = self.modules_json.get_module_version(module, modules_repo.remote_url) # Set the temporary installation folder install_dir = Path(tempfile.mkdtemp()) @@ -303,7 +303,7 @@ def get_single_module_info(self, module): log.info(f"Updating module to ({sha})") # Check if the update branch is the same as the installation branch - current_branch = self.modules_json.get_module_branch(module, self.modules_repo.fullname) + current_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url) new_branch = self.modules_repo.branch if current_branch != new_branch: log.warning( diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 7533696f99..d6a991dadb 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -2,9 +2,8 @@ "name": "{{ name }}", "homePage": "https://github.com/{{ name }}", "repos": { - "nf-core/modules": { - "git_url": "https://github.com/nf-core/modules.git", - "modules": { + "https://github.com/nf-core/modules.git": { + "modules/nf-core": { "custom/dumpsoftwareversions": { "branch": "master", "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" diff --git a/tests/modules/install.py b/tests/modules/install.py index d2b13c2aee..449db055c3 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -66,4 +66,4 @@ def test_modules_install_different_branch_succeed(self): # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 2412f9bd2d..1c6ad0ff31 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -188,10 +188,10 @@ def test_mod_json_get_module_version(self): mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() assert ( - mod_json_obj.get_module_version("fastqc", NF_CORE_MODULES_NAME) - == mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"]["git_sha"] + mod_json_obj.get_module_version("fastqc", NF_CORE_MODULES_REMOTE) + == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["fastqc"]["git_sha"] ) - assert mod_json_obj.get_module_version("INVALID_MODULE", NF_CORE_MODULES_NAME) is None + assert mod_json_obj.get_module_version("INVALID_MODULE", NF_CORE_MODULES_REMOTE) is None def test_mod_json_get_git_url(self): diff --git a/tests/modules/update.py b/tests/modules/update.py index f49d2be257..8610ffd7f5 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -201,8 +201,8 @@ def test_update_different_branch_single_module(self): # Verify that the branch entry was updated correctly modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH - assert modules_json.get_module_version("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + assert modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA def test_update_different_branch_mixed_modules_main(self): @@ -223,10 +223,10 @@ def test_update_different_branch_mixed_modules_main(self): modules_json = ModulesJson(self.pipeline_dir) # Verify that the branch entry was updated correctly - assert modules_json.get_module_branch("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH - assert modules_json.get_module_version("fastp", GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + assert modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA # MultiQC is present in both branches but should've been updated using the 'main' branch - assert modules_json.get_module_branch("multiqc", GITLAB_REPO) == GITLAB_DEFAULT_BRANCH + assert modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_DEFAULT_BRANCH def test_update_different_branch_mix_modules_branch_test(self): @@ -240,8 +240,8 @@ def test_update_different_branch_mix_modules_branch_test(self): update_obj.update() modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("multiqc", GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH - assert modules_json.get_module_version("multiqc", GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + assert modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("multiqc", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA def cmp_module(dir1, dir2): From d5eb924ff8b967b17aa1bd4938164f6d5528a95a Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 30 Sep 2022 10:43:00 +0200 Subject: [PATCH 152/854] update subworkflow template --- nf_core/subworkflow-template/subworkflows/main.nf | 4 ++++ nf_core/subworkflow-template/tests/main.nf | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/subworkflow-template/subworkflows/main.nf b/nf_core/subworkflow-template/subworkflows/main.nf index eac79c8cff..05b3b2ec64 100644 --- a/nf_core/subworkflow-template/subworkflows/main.nf +++ b/nf_core/subworkflow-template/subworkflows/main.nf @@ -3,6 +3,10 @@ // You can also ask for help via your pull request or on the #subworkflows channel on the nf-core Slack workspace: // https://nf-co.re/join // TODO nf-core: A subworkflow SHOULD only import modules not other subworkflows +// TODO nf-core: A subworkflow SHOULD import at least two modules + +include { SAMTOOLS_SORT } from '../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' workflow {{ subworkflow_name|upper }} { diff --git a/nf_core/subworkflow-template/tests/main.nf b/nf_core/subworkflow-template/tests/main.nf index 0dd0b8f890..10d584156b 100644 --- a/nf_core/subworkflow-template/tests/main.nf +++ b/nf_core/subworkflow-template/tests/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { {{ subworkflow_name|upper }} } from '../../../{{ "../" if subtool else "" }}subworkflows/{{ subworkflow_dir }}/main.nf' +include { {{ subworkflow_name|upper }} } from '../../../subworkflows/nf-core/{{ subworkflow_dir }}/main.nf' workflow test_{{ subworkflow_name }} { {% if has_meta %} From 9ce3e57875448c145cfbb989c5baf214873b7113 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 30 Sep 2022 10:55:39 +0200 Subject: [PATCH 153/854] fix paths create --- CHANGELOG.md | 1 + nf_core/subworkflow-template/tests/main.nf | 2 +- nf_core/subworkflows/create.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 920ba98be4..cf952b40f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Schema: Remove `allOf` if no definition groups are left. - Use contextlib to temporarily change working directories ([#1819](https://github.com/nf-core/tools/pull/1819)) - More helpful error messages if `nf-core download` can't parse a singularity image download +- Add `nf-core subworkflows create` command ### Modules diff --git a/nf_core/subworkflow-template/tests/main.nf b/nf_core/subworkflow-template/tests/main.nf index 10d584156b..7d9b71f58a 100644 --- a/nf_core/subworkflow-template/tests/main.nf +++ b/nf_core/subworkflow-template/tests/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { {{ subworkflow_name|upper }} } from '../../../subworkflows/nf-core/{{ subworkflow_dir }}/main.nf' +include { {{ subworkflow_name|upper }} } from '../../../../subworkflows/nf-core/{{ subworkflow_dir }}/main.nf' workflow test_{{ subworkflow_name }} { {% if has_meta %} diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 533476ecbe..0a9a63f89a 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -53,10 +53,10 @@ def create(self): If is a clone of nf-core/modules, it creates or modifies the following files: - subworkflows/subworkflow_name/ + subworkflows/nf-core/subworkflow_name/ * main.nf * meta.yml - tests/subworkflows/subworkflow_name/ + tests/subworkflows/nf-core/subworkflow_name/ * main.nf * test.yml * nextflow.config From 656a74b38f698385e6f8ae2367519d15a25f92ce Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 30 Sep 2022 11:16:42 +0200 Subject: [PATCH 154/854] fix more paths --- nf_core/subworkflow-template/subworkflows/main.nf | 4 ++-- nf_core/subworkflow-template/tests/test.yml | 10 +++++----- nf_core/subworkflows/create.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nf_core/subworkflow-template/subworkflows/main.nf b/nf_core/subworkflow-template/subworkflows/main.nf index 05b3b2ec64..7895f8aa97 100644 --- a/nf_core/subworkflow-template/subworkflows/main.nf +++ b/nf_core/subworkflow-template/subworkflows/main.nf @@ -5,8 +5,8 @@ // TODO nf-core: A subworkflow SHOULD only import modules not other subworkflows // TODO nf-core: A subworkflow SHOULD import at least two modules -include { SAMTOOLS_SORT } from '../../modules/nf-core/samtools/sort/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' +include { SAMTOOLS_SORT } from '../../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' workflow {{ subworkflow_name|upper }} { diff --git a/nf_core/subworkflow-template/tests/test.yml b/nf_core/subworkflow-template/tests/test.yml index d3650dafae..4462e7107a 100644 --- a/nf_core/subworkflow-template/tests/test.yml +++ b/nf_core/subworkflow-template/tests/test.yml @@ -1,12 +1,12 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core subworkflows create-test-yml {{ subworkflow_name_underscore }} -- name: "{{ subworkflow_name_underscore }}" - command: nextflow run ./tests/subworkflows/{{ subworkflow_dir }} -entry test_{{ subworkflow_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/subworkflows/{{ subworkflow_dir }}/nextflow.config +- name: "{{ subworkflow_name }}" + command: nextflow run ./tests/subworkflows/nf-core/{{ subworkflow_dir }} -entry test_{{ subworkflow_name }} -c ./tests/config/nextflow.config -c ./tests/subworkflows/nf-core/{{ subworkflow_dir }}/nextflow.config tags: - - "{{ subworkflow_name_underscore }}" + - "{{ subworkflow_name }}" files: - - path: "output/{{ subworkflow_name_underscore }}/test.bam" + - path: "output/{{ subworkflow_name }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: output/{{ subworkflow_name_underscore }}/versions.yml + - path: output/{{ subworkflow_name }}/versions.yml md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 0a9a63f89a..9ab91fd2ac 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -132,8 +132,8 @@ def create(self): with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: pytest_modules_yml = yaml.safe_load(fh) pytest_modules_yml[self.subworkflow] = [ - f"subworkflows/{self.subworkflow}/**", - f"tests/subworkflows/{self.subworkflow}/**", + f"subworkflows/nf-core/{self.subworkflow}/**", + f"tests/subworkflows/nf-core/{self.subworkflow}/**", ] pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: From 2433fe8f9449da936a4ffc847da9b9a53c033eee Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 30 Sep 2022 17:49:54 +0200 Subject: [PATCH 155/854] modify modules structure with two levels modules/nf-core or modules/custom_dir --- nf_core/lint/modules_json.py | 27 +-- nf_core/modules/info.py | 15 +- nf_core/modules/install.py | 4 +- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/lint/module_version.py | 2 +- nf_core/modules/list.py | 30 ++- nf_core/modules/module_utils.py | 6 + nf_core/modules/modules_command.py | 8 +- nf_core/modules/modules_json.py | 279 +++++++++++++------------ nf_core/modules/modules_repo.py | 6 +- nf_core/modules/patch.py | 37 ++-- nf_core/modules/remove.py | 12 +- nf_core/modules/update.py | 130 +++++++----- nf_core/pipeline-template/modules.json | 26 +-- tests/modules/install.py | 4 +- tests/modules/patch.py | 20 +- tests/modules/update.py | 16 +- 17 files changed, 365 insertions(+), 259 deletions(-) diff --git a/nf_core/lint/modules_json.py b/nf_core/lint/modules_json.py index 475113fa00..3e97eed043 100644 --- a/nf_core/lint/modules_json.py +++ b/nf_core/lint/modules_json.py @@ -21,31 +21,32 @@ def modules_json(self): _modules_json = ModulesJson(self.wf_path) _modules_json.load() modules_json_dict = _modules_json.modules_json - modules_dir = Path(self.wf_path, "modules", "nf-core") + modules_dir = Path(self.wf_path, "modules") if _modules_json: all_modules_passed = True for repo in modules_json_dict["repos"].keys(): # Check if the modules.json has been updated to keep the - if "modules/nf-core" not in modules_json_dict["repos"][repo] or not repo.startswith("http"): + if "modules" not in modules_json_dict["repos"][repo] or not repo.startswith("http"): failed.append( "Your `modules.json` file is outdated. " "It will be automatically generated by running any module command." ) continue - for module, module_entry in modules_json_dict["repos"][repo]["modules/nf-core"].items(): - if not Path(modules_dir, module).exists(): - failed.append( - f"Entry for `{Path(modules_dir, module)}` found in `modules.json` but module is not installed in " - "pipeline." - ) - all_modules_passed = False - if module_entry.get("branch") is None: - failed.append(f"Entry for `{Path(modules_dir, module)}` is missing branch information.") - if module_entry.get("git_sha") is None: - failed.append(f"Entry for `{Path(modules_dir, module)}` is missing version information.") + for dir in modules_json_dict["repos"][repo]["modules"].keys(): + for module, module_entry in modules_json_dict["repos"][repo]["modules"][dir].items(): + if not Path(modules_dir, dir, module).exists(): + failed.append( + f"Entry for `{Path(modules_dir, dir, module)}` found in `modules.json` but module is not installed in " + "pipeline." + ) + all_modules_passed = False + if module_entry.get("branch") is None: + failed.append(f"Entry for `{Path(modules_dir, dir, module)}` is missing branch information.") + if module_entry.get("git_sha") is None: + failed.append(f"Entry for `{Path(modules_dir, dir, module)}` is missing version information.") if all_modules_passed: passed.append("Only installed modules found in `modules.json`") else: diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 37e0b4db43..dfc4431df3 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -93,7 +93,8 @@ def init_mod_name(self, module): if self.repo_type == "modules": modules = self.get_modules_clone_modules() else: - modules = self.modules_json.get_all_modules().get(self.modules_repo.fullname) + modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) + modules = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] if modules is None: raise UserWarning(f"No modules installed from '{self.modules_repo.remote_url}'") else: @@ -136,14 +137,16 @@ def get_local_yaml(self): if self.repo_type == "pipeline": # Try to find and load the meta.yml file repo_name = self.modules_repo.fullname - module_base_path = os.path.join(self.dir, "modules", repo_name) + module_base_path = os.path.join(self.dir, "modules") # Check that we have any modules installed from this repo - modules = self.modules_json.get_all_modules().get(repo_name) + modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) + module_names = [module for _, module in modules] if modules is None: raise LookupError(f"No modules installed from {self.modules_repo.remote_url}") - if self.module in modules: - mod_dir = os.path.join(module_base_path, self.module) + if self.module in module_names: + install_dir = [dir for dir, module in modules if module == self.module][0] + mod_dir = os.path.join(module_base_path, install_dir, self.module) meta_fn = os.path.join(mod_dir, "meta.yml") if os.path.exists(meta_fn): log.debug(f"Found local file: {meta_fn}") @@ -153,7 +156,7 @@ def get_local_yaml(self): log.debug(f"Module '{self.module}' meta.yml not found locally") else: - module_base_path = os.path.join(self.dir, "modules") + module_base_path = os.path.join(self.dir, "modules", "nf-core") if self.module in os.listdir(module_base_path): mod_dir = os.path.join(module_base_path, self.module) meta_fn = os.path.join(mod_dir, "meta.yml") diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 07cb01d406..54f0a94f20 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -71,7 +71,9 @@ def install(self, module): log.warning(warn_msg) return False - current_version = modules_json.get_module_version(module, self.modules_repo.remote_url) + current_version = modules_json.get_module_version( + module, self.modules_repo.remote_url, self.modules_repo.fullname + ) # Set the install folder based on the repository name install_folder = os.path.join(self.dir, "modules", self.modules_repo.fullname) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index c4d439812e..83152ac222 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -144,7 +144,7 @@ def lint( Lint all or one specific module First gets a list of all local modules (in modules/local/process) and all modules - installed from nf-core (in modules/nf-core/modules) + installed from nf-core (in modules/nf-core) For all nf-core modules, the correct file structure is assured and important file content is verified. If directory subject to linting is a clone of 'nf-core/modules', diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index 8f179ba8db..d2b7b1524d 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -26,7 +26,7 @@ def module_version(module_lint_object, module): # Verify that a git_sha exists in the `modules.json` file for this module version = module_lint_object.modules_json.get_module_version( - module.module_name, module_lint_object.modules_repo.remote_url + module.module_name, module_lint_object.modules_repo.remote_url, module_lint_object.modules_repo.fullname ) if version is None: module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path)) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index f063f21151..046efb1d7e 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -3,6 +3,8 @@ import rich +import nf_core.modules.module_utils + from .modules_command import ModuleCommand from .modules_json import ModulesJson from .modules_repo import ModulesRepo @@ -57,6 +59,16 @@ def pattern_msg(keywords): # We have a pipeline - list what's installed else: + # Check that we are in a pipeline directory + try: + _, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + if repo_type != "pipeline": + raise UserWarning( + "The command 'nf-core modules list local' must be run from a pipeline directory.", + ) + except UserWarning as e: + log.error(e) + return "" # Check whether pipelines is valid try: self.has_valid_directory() @@ -70,8 +82,8 @@ def pattern_msg(keywords): # Filter by keywords repos_with_mods = { - repo_name: [mod for mod in modules if all(k in mod for k in keywords)] - for repo_name, modules in modules_json.get_all_modules().items() + repo_url: [mod for mod in modules if all(k in mod[1] for k in keywords)] + for repo_url, modules in modules_json.get_all_modules().items() } # Nothing found @@ -87,18 +99,18 @@ def pattern_msg(keywords): # Load 'modules.json' modules_json = modules_json.modules_json - for repo_name, modules in sorted(repos_with_mods.items()): - repo_entry = modules_json["repos"].get(repo_name, {}) - for module in sorted(modules): + for repo_url, module_with_dir in sorted(repos_with_mods.items()): + repo_entry = modules_json["repos"].get(repo_url, {}) + for install_dir, module in sorted(module_with_dir): repo_modules = repo_entry.get("modules") - module_entry = repo_modules.get(module) + module_entry = repo_modules.get(install_dir).get(module) if module_entry: version_sha = module_entry["git_sha"] try: # pass repo_name to get info on modules even outside nf-core/modules message, date = ModulesRepo( - remote_url=repo_entry["git_url"], + remote_url=repo_url, branch=module_entry["branch"], ).get_commit_info(version_sha) except LookupError as e: @@ -106,11 +118,11 @@ def pattern_msg(keywords): date = "[red]Not Available" message = "[red]Not Available" else: - log.warning(f"Commit SHA for module '{repo_name}/{module}' is missing from 'modules.json'") + log.warning(f"Commit SHA for module '{install_dir}/{module}' is missing from 'modules.json'") version_sha = "[red]Not Available" date = "[red]Not Available" message = "[red]Not Available" - table.add_row(module, repo_name, version_sha, message, date) + table.add_row(module, repo_url, version_sha, message, date) if print_json: return json.dumps(modules, sort_keys=True, indent=4) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 3bfa20b6a8..23315cc7f1 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -30,14 +30,20 @@ def path_from_remote(remote_url): path = path.path # Remove the intial '/' path = path[1:] + # Remove extension path = os.path.splitext(path)[0] + # Remove repo name "modules" + path = os.path.split(path)[0] else: # Remove the initial `git@`` path = remote_url.split("@") path = path[-1] if len(path) > 1 else path[0] path = urllib.parse.urlparse(path) path = path.path + # Remove extension path = os.path.splitext(path)[0] + # Remove repo name "modules" + path = os.path.split(path)[0] return path diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 9dfffa6e9b..9c19799bff 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -93,19 +93,19 @@ def clear_module_dir(self, module_name, module_dir): log.error(f"Could not remove module: {e}") return False - def modules_from_repo(self, repo_name): + def modules_from_repo(self, install_dir): """ Gets the modules installed from a certain repository Args: - repo_name (str): The name of the repository + install_dir (str): The name of the directory where modules are installed Returns: [str]: The names of the modules """ - repo_dir = Path(self.dir, "modules", repo_name) + repo_dir = Path(self.dir, "modules", install_dir) if not repo_dir.exists(): - raise LookupError(f"Nothing installed from {repo_name} in pipeline") + raise LookupError(f"Nothing installed from {install_dir} in pipeline") return [ str(Path(dir_path).relative_to(repo_dir)) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 904d74e5a7..636f3491ed 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -33,7 +33,7 @@ def __init__(self, pipeline_dir): pipeline_dir (str): The pipeline directory """ self.dir = pipeline_dir - self.modules_dir = Path(self.dir, "modules", "nf-core") + self.modules_dir = Path(self.dir, "modules") self.modules_json = None self.pipeline_modules = None @@ -48,7 +48,7 @@ def create(self): pipeline_name = pipeline_config.get("manifest.name", "") pipeline_url = pipeline_config.get("manifest.homePage", "") modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": {}} - modules_dir = Path(self.dir, "modules/nf-core") + modules_dir = Path(self.dir, "modules") if not modules_dir.exists(): raise UserWarning("Can't find a ./modules directory. Is this a DSL2 pipeline?") @@ -58,22 +58,24 @@ def create(self): # Get all module names in the repos repo_module_names = [ ( - repo_name, + repo_url, [ - str(Path(dir_name).relative_to(modules_dir / repo_name)) - for dir_name, _, file_names in os.walk(modules_dir / repo_name) + str(Path(module_name).relative_to(modules_dir / dir)) + for module_name, _, file_names in os.walk(modules_dir / dir) if "main.nf" in file_names ], - repo_remote, + dir, ) - for repo_name, repo_remote in repos.items() + for dir, modules in repo_dict["modules"].items() + for repo_url, repo_dict in repos.items() ] - for repo_name, module_names, remote_url in sorted(repo_module_names): - modules_json["repos"][remote_url] = {} - modules_json["repos"][remote_url]["modules/nf-core"] = {} - modules_json["repos"][remote_url]["modules/nf-core"] = self.determine_module_branches_and_shas( - repo_name, remote_url, module_names + for repo_url, module_names, install_dir in sorted(repo_module_names): + modules_json["repos"][repo_url] = {} + modules_json["repos"][repo_url]["modules"] = {} + modules_json["repos"][repo_url]["modules"][install_dir] = {} + modules_json["repos"][repo_url]["modules"] = self.determine_module_branches_and_shas( + install_dir, repo_url, module_names ) # write the modules.json file and assign it to the object modules_json_path = Path(self.dir, "modules.json") @@ -100,13 +102,13 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): # Check if there are any nf-core modules installed if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): - repos[ - nf_core.modules.modules_repo.NF_CORE_MODULES_NAME - ] = nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE + repos[nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE] = {} # The function might rename some directories, keep track of them renamed_dirs = {} # Check if there are any untracked repositories - dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos]) + dirs_not_covered = self.dir_tree_uncovered( + modules_dir, [Path(name) for url in repos for name in repos[url][modules_dir]] + ) if len(dirs_not_covered) > 0: log.info("Found custom module repositories when creating 'modules.json'") # Loop until all directories in the base directory are covered by a remote @@ -150,8 +152,10 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): else: continue - repos[nrepo_name] = (nrepo_remote, "modules/nf-core") - dirs_not_covered = self.dir_tree_uncovered(modules_dir, [Path(name) for name in repos]) + repos[nrepo_remote]["modules"][nrepo_name] = {} + dirs_not_covered = self.dir_tree_uncovered( + modules_dir, [Path(name) for name in repos[url][modules_dir] for url in repos] + ) return repos, renamed_dirs def dir_tree_uncovered(self, modules_dir, repos): @@ -189,7 +193,7 @@ def dir_tree_uncovered(self, modules_dir, repos): depth += 1 return dirs_not_covered - def determine_module_branches_and_shas(self, repo_name, remote_url, modules): + def determine_module_branches_and_shas(self, install_dir, remote_url, modules): """ Determines what branch and commit sha each module in the pipeline belong to @@ -197,7 +201,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, modules): module in the default branch, it prompts the user with the available branches Args: - repo_name (str): The name of the module repository + install_dir (str): The name of the directory inside modules where modules are installed remote_url (str): The url to the remote repository modules ([str]): List of names of installed modules from the repository @@ -206,7 +210,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, modules): from the repository """ default_modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url) - repo_path = self.modules_dir / repo_name + repo_path = self.modules_dir / install_dir # Get the branches present in the repository, as well as the default branch available_branches = nf_core.modules.modules_repo.ModulesRepo.get_remote_branches(remote_url) sb_local = [] @@ -222,7 +226,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, modules): # If the module is patched patch_file = module_path / f"{module}.diff" if patch_file.is_file(): - temp_module_dir = self.try_apply_patch_reverse(module, repo_name, patch_file, module_path) + temp_module_dir = self.try_apply_patch_reverse(module, install_dir, patch_file, module_path) correct_commit_sha = self.find_correct_commit_sha(module, temp_module_dir, modules_repo) else: correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo) @@ -262,11 +266,11 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, modules): # Clean up the modules we were unable to find the sha for for module in sb_local: - log.debug(f"Moving module '{Path(repo_name, module)}' to 'local' directory") - self.move_module_to_local(module, repo_name) + log.debug(f"Moving module '{Path(install_dir, module)}' to 'local' directory") + self.move_module_to_local(module, install_dir) for module in dead_modules: - log.debug(f"Removing module {Path(repo_name, module)}'") + log.debug(f"Removing module {Path(install_dir, module)}'") shutil.rmtree(repo_path / module) return repo_entry @@ -323,7 +327,9 @@ def unsynced_modules(self): by the modules.json file, and modules in the modules.json where the installation directory is missing """ + # Add all modules from modules.json to missing_installation missing_installation = copy.deepcopy(self.modules_json["repos"]) + # Obtain the path of all installed modules dirs = [ Path(dir_name).relative_to(self.modules_dir) for dir_name, _, file_names in os.walk(self.modules_dir) @@ -331,32 +337,32 @@ def unsynced_modules(self): ] untracked_dirs = [] for dir in dirs: - # Check if the modules directory exists - module_repo_name = None + # Check if the modules directory exists in modules.json + install_dir = dir.parts[0] + module = str(Path(*dir.parts[1:])) + module_in_file = False git_url = None for repo in missing_installation: - for dir_name in missing_installation[repo]: - module_repo_name = dir_name - git_url = repo - break - if module_repo_name is not None: - # If it does, check if the module is in the 'modules.json' file - module = str(dir) - module_repo = missing_installation[git_url] - - if module not in module_repo["modules/nf-core"].keys(): - untracked_dirs.append(dir) - else: - # Check if the entry has a git sha and branch before removing - modules = module_repo["modules/nf-core"] - if "git_sha" not in modules[module] or "branch" not in modules[module]: - self.determine_module_branches_and_shas(module, git_url, module_repo["base_path"], [module]) - module_repo["modules/nf-core"].pop(module) - if len(module_repo["modules/nf-core"]) == 0: - missing_installation.pop(git_url) - else: + for dir_name in missing_installation[repo]["modules"]: + if module in missing_installation[repo]["modules"][dir_name]: + module_in_file = True + git_url = repo + break + if not module_in_file: # If it is not, add it to the list of missing modules - untracked_dirs.append(dir) + untracked_dirs.append(module) + else: + # If it does, remove the module from missing_installation + module_repo = missing_installation[git_url] + # Check if the entry has a git sha and branch before removing + modules = module_repo["modules"][install_dir] + if "git_sha" not in modules[module] or "branch" not in modules[module]: + self.determine_module_branches_and_shas(module, git_url, module_repo["base_path"], [module]) + # Remove the module from modules without installation + module_repo["modules"][install_dir].pop(module) + if len(module_repo["modules"][install_dir]) == 0: + # If no modules with missing installation left, remove the git_url from missing_installation + missing_installation.pop(git_url) return untracked_dirs, missing_installation @@ -368,25 +374,25 @@ def has_git_url_and_modules(self): (bool): True if they are found for all repos, False otherwise """ for repo_url, repo_entry in self.modules_json.get("repos", {}).items(): - if "modules/nf-core" not in repo_entry: + if "modules" not in repo_entry: log.warning(f"modules.json entry {repo_entry} does not have a modules entry") return False elif ( not isinstance(repo_url, str) or repo_url == "" - or not isinstance(repo_entry["modules/nf-core"], dict) - or repo_entry["modules/nf-core"] == {} + or not isinstance(repo_entry["modules"], dict) + or repo_entry["modules"] == {} ): log.warning(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules") return False return True - def reinstall_repo(self, repo_name, remote_url, module_entries): + def reinstall_repo(self, install_dir, remote_url, module_entries): """ Reinstall modules from a repository Args: - repo_name (str): The name of the repository + install_dir (str): The name of directory where modules are installed remote_url (str): The git url of the remote repository modules ([ dict[str, dict[str, str]] ]): Module entries with branch and git sha info @@ -413,8 +419,10 @@ def reinstall_repo(self, repo_name, remote_url, module_entries): log.error(e) failed_to_install.extend(modules) for module, sha in modules: - if not modules_repo.install_module(module, self.modules_dir, sha): - log.warning(f"Could not install module '{Path(repo_name, module)}' - removing from modules.json") + if not modules_repo.install_module(module, self.modules_dir / install_dir, sha): + log.warning( + f"Could not install module '{Path(self.modules_dir, install_dir, module)}' - removing from modules.json" + ) failed_to_install.append(module) return failed_to_install @@ -444,24 +452,28 @@ def check_up_to_date(self): # we try to reinstall them if len(missing_installation) > 0: missing_but_in_mod_json = [ - f"'modules/nf-core/{module}'" + f"'modules/{install_dir}/{module}'" for repo_url, contents in missing_installation.items() - for module in contents["modules/nf-core"] + for install_dir, dir_contents in contents["modules"].items() + for module in dir_contents ] log.info( f"Reinstalling modules found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" ) remove_from_mod_json = {} - for repo, contents in missing_installation.items(): - module_entries = contents["modules/nf-core"] - remote_url = repo - remove_from_mod_json[repo] = self.reinstall_repo(repo, remote_url, module_entries) + for repo_url, contents in missing_installation.items(): + for install_dir, module_entries in contents["modules"].items(): + remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( + install_dir, repo_url, module_entries + ) # If the reinstall fails, we remove those entries in 'modules.json' if sum(map(len, remove_from_mod_json.values())) > 0: uninstallable_mods = [ - f"'{repo}/{module}'" for repo, modules in remove_from_mod_json.items() for module in modules + f"'{install_dir}/{module}'" + for (repo_url, install_dir), modules in remove_from_mod_json.items() + for module in modules ] if len(uninstallable_mods) == 1: log.info(f"Was unable to reinstall {uninstallable_mods[0]}. Removing 'modules.json' entry") @@ -470,11 +482,11 @@ def check_up_to_date(self): f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_mods)}" ) - for repo, module_entries in remove_from_mod_json.items(): + for (repo_url, install_dir), module_entries in remove_from_mod_json.items(): for module in module_entries: - self.modules_json["repos"][repo]["modules/nf-core"].pop(module) - if len(self.modules_json["repos"][repo]["modules/nf-core"]) == 0: - self.modules_json["repos"].pop(repo) + self.modules_json["repos"][repo_url]["modules"][install_dir].pop(module) + if len(self.modules_json["repos"][repo_url]["modules"][install_dir]) == 0: + self.modules_json["repos"].pop(repo_url) # If some modules didn't have an entry in the 'modules.json' file # we try to determine the SHA from the commit log of the remote @@ -488,30 +500,33 @@ def check_up_to_date(self): ) # Get the remotes we are missing - tracked_repos = {repo_name: (remote_url) for repo_name, repo_entry in self.modules_json["repos"].items()} + tracked_repos = {repo_url: (repo_entry) for repo_url, repo_entry in self.modules_json["repos"].items()} repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) modules_with_repos = ( - (repo_name, str(dir.relative_to(repo_name))) + (install_dir, str(dir.relative_to(install_dir))) for dir in missing_from_modules_json - for repo_name in repos - if nf_core.utils.is_relative_to(dir, repo_name) + for repo_url, repo_content in repos.items() + for install_dir, modules in repo_content["modules"].items() + if nf_core.utils.is_relative_to(dir, install_dir) ) repos_with_modules = {} - for repo_name, module in modules_with_repos: - if repo_name not in repos_with_modules: - repos_with_modules[repo_name] = [] - repos_with_modules[repo_name].append(module) - - for repo_name, modules in repos_with_modules.items(): - remote_url = repos[repo_name] - repo_entry = self.determine_module_branches_and_shas(repo_name, remote_url, modules) - if repo_name in self.modules_json["repos"]: - self.modules_json["repos"][repo_name]["modules/nf-core"].update(repo_entry) + for install_dir, module in modules_with_repos: + if install_dir not in repos_with_modules: + repos_with_modules[install_dir] = [] + repos_with_modules[install_dir].append(module) + + for install_dir, modules in repos_with_modules.items(): + remote_url = [url for url, content in repos.items() if install_dir in content][0] + repo_entry = self.determine_module_branches_and_shas(install_dir, remote_url, modules) + if remote_url in self.modules_json["repos"]: + self.modules_json["repos"][remote_url]["modules"][install_dir].update(repo_entry) else: - self.modules_json["repos"][repo_name] = { - "modules/nf-core": repo_entry, + self.modules_json["repos"][remote_url] = { + "modules": { + install_dir: repo_entry, + } } self.dump() @@ -547,9 +562,9 @@ def update(self, modules_repo, module_name, module_version, write_file=True): repo_name = modules_repo.fullname remote_url = modules_repo.remote_url branch = modules_repo.branch - if repo_name not in self.modules_json["repos"]: - self.modules_json["repos"][repo_name] = {"modules/nf-core": {}} - repo_modules_entry = self.modules_json["repos"][repo_name]["modules/nf-core"] + if remote_url not in self.modules_json["repos"]: + self.modules_json["repos"][remote_url] = {"modules": {repo_name: {}}} + repo_modules_entry = self.modules_json["repos"][remote_url]["modules"][repo_name] if module_name not in repo_modules_entry: repo_modules_entry[module_name] = {} repo_modules_entry[module_name]["git_sha"] = module_version @@ -560,62 +575,71 @@ def update(self, modules_repo, module_name, module_version, write_file=True): if write_file: self.dump() - def remove_entry(self, module_name, repo_name): + def remove_entry(self, module_name, repo_url, install_dir): """ Removes an entry from the 'modules.json' file. Args: module_name (str): Name of the module to be removed - repo_name (str): Name of the repository containing the module + repo_url (str): URL of the repository containing the module + install_dir (str): Name of the directory where modules are installed Returns: (bool): True if the removal was successful, False otherwise """ if not self.modules_json: return False - if repo_name in self.modules_json.get("repos", {}): - repo_entry = self.modules_json["repos"][repo_name] - if module_name in repo_entry.get("modules/nf-core", {}): - repo_entry["modules/nf-core"].pop(module_name) + if repo_url in self.modules_json.get("repos", {}): + repo_entry = self.modules_json["repos"][repo_url] + if module_name in repo_entry["modules"].get(install_dir, {}): + repo_entry["modules"][install_dir].pop(module_name) else: - log.warning(f"Module '{repo_name}/{module_name}' is missing from 'modules.json' file.") + log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.") return False - if len(repo_entry["modules/nf-core"]) == 0: - self.modules_json["repos"].pop(repo_name) + if len(repo_entry["modules"][install_dir]) == 0: + self.modules_json["repos"].pop(repo_url) else: - log.warning(f"Module '{repo_name}/{module_name}' is missing from 'modules.json' file.") + log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.") return False self.dump() return True - def add_patch_entry(self, module_name, repo_name, patch_filename, write_file=True): + def add_patch_entry(self, module_name, repo_url, install_dir, patch_filename, write_file=True): """ Adds (or replaces) the patch entry for a module """ if self.modules_json is None: self.load() - if repo_name not in self.modules_json["repos"]: - raise LookupError(f"Repo '{repo_name}' not present in 'modules.json'") - if module_name not in self.modules_json["repos"][repo_name]["modules/nf-core"]: - raise LookupError(f"Module '{repo_name}/{module_name}' not present in 'modules.json'") - self.modules_json["repos"][repo_name]["modules/nf-core"][module_name]["patch"] = str(patch_filename) + if repo_url not in self.modules_json["repos"]: + raise LookupError(f"Repo '{repo_url}' not present in 'modules.json'") + if module_name not in self.modules_json["repos"][repo_url]["modules"][install_dir]: + raise LookupError(f"Module '{install_dir}/{module_name}' not present in 'modules.json'") + self.modules_json["repos"][repo_url]["modules"][install_dir][module_name]["patch"] = str(patch_filename) if write_file: self.dump() - def get_patch_fn(self, module_name, repo_name): + def get_patch_fn(self, module_name, repo_url, install_dir): """ Get the patch filename of a module Args: module_name (str): The name of the module - repo_name (str): The name of the repository containing the module + repo_url (str): The URL of the repository containing the module + install_dir (str): The name of the directory where modules are installed Returns: (str): The patch filename for the module, None if not present """ if self.modules_json is None: self.load() - path = self.modules_json["repos"].get(repo_name, {}).get("modules/nf-core").get(module_name, {}).get("patch") + path = ( + self.modules_json["repos"] + .get(repo_url, {}) + .get("modules") + .get(install_dir) + .get(module_name, {}) + .get("patch") + ) return Path(path) if path is not None else None def try_apply_patch_reverse(self, module, repo_name, patch_relpath, module_dir): @@ -666,18 +690,21 @@ def repo_present(self, repo_name): self.load() return repo_name in self.modules_json.get("repos", {}) - def module_present(self, module_name, repo_name): + def module_present(self, module_name, repo_url, install_dir): """ Checks if a module is present in the modules.json file Args: module_name (str): Name of the module - repo_name (str): Name of the repository + repo_url (str): URL of the repository + install_dir (str): Name of the directory where modules are installed Returns: (bool): Whether the module is present in the 'modules.json' file """ if self.modules_json is None: self.load() - return module_name in self.modules_json.get("repos", {}).get(repo_name, {}).get("modules/nf-core", {}) + return module_name in self.modules_json.get("repos", {}).get(repo_url, {}).get("modules", {}).get( + install_dir, {} + ) def get_modules_json(self): """ @@ -690,13 +717,14 @@ def get_modules_json(self): self.load() return copy.deepcopy(self.modules_json) - def get_module_version(self, module_name, repo_url): + def get_module_version(self, module_name, repo_url, install_dir): """ Returns the version of a module Args: module_name (str): Name of the module repo_url (str): URL of the repository + install_dir (str): Name of the directory where modules are installed Returns: (str): The git SHA of the module if it exists, None otherwise @@ -706,25 +734,12 @@ def get_module_version(self, module_name, repo_url): return ( self.modules_json.get("repos", {}) .get(repo_url, {}) - .get("modules/nf-core", {}) + .get("modules", {}) + .get(install_dir, {}) .get(module_name, {}) .get("git_sha", None) ) - # def get_git_url(self, repo_name): - # """ - # Returns the git url of a repo - - # Args: - # repo_name (str): Name of the repository - - # Returns: - # (str): The git url of the repository if it exists, None otherwise - # """ - # if self.modules_json is None: - # self.load() - # return self.modules_json.get("repos", {}).get(repo_name, {}).get("git_url", None) - def get_all_modules(self): """ Retrieves all pipeline modules that are reported in the modules.json @@ -738,12 +753,13 @@ def get_all_modules(self): if self.pipeline_modules is None: self.pipeline_modules = {} for repo, repo_entry in self.modules_json.get("repos", {}).items(): - if "modules/nf-core" in repo_entry: - self.pipeline_modules[repo] = list(repo_entry["modules/nf-core"]) + if "modules" in repo_entry: + for dir, modules in repo_entry["modules"].items(): + self.pipeline_modules[repo] = [(dir, m) for m in modules] return self.pipeline_modules - def get_module_branch(self, module, repo_url): + def get_module_branch(self, module, repo_url, install_dir): """ Gets the branch from which the module was installed @@ -754,10 +770,17 @@ def get_module_branch(self, module, repo_url): """ if self.modules_json is None: self.load() - branch = self.modules_json["repos"].get(repo_url, {}).get("modules/nf-core", {}).get(module, {}).get("branch") + branch = ( + self.modules_json["repos"] + .get(repo_url, {}) + .get("modules", {}) + .get(install_dir, {}) + .get(module, {}) + .get("branch") + ) if branch is None: raise LookupError( - f"Could not find branch information for module '{Path('modules/nf-core', module)}'." + f"Could not find branch information for module '{Path(install_dir, module)}'." f"Please remove the 'modules.json' and rerun the command to recreate it" ) return branch diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 1bb2770f33..e21048077f 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) # Constants for the nf-core/modules repo used throughout the module files -NF_CORE_MODULES_NAME = "nf-core/modules" +NF_CORE_MODULES_NAME = "nf-core" NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" NF_CORE_MODULES_DEFAULT_BRANCH = "master" @@ -149,7 +149,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True): branch (str): name of branch to use Sets self.repo """ - self.local_repo_dir = os.path.join(NFCORE_DIR, self.fullname) + self.local_repo_dir = os.path.join(NFCORE_DIR, self.fullname, "modules") if not os.path.exists(self.local_repo_dir): try: pbar = rich.progress.Progress( @@ -313,7 +313,7 @@ def install_module(self, module_name, install_dir, commit): return False # Copy the files from the repo to the install folder - shutil.copytree(self.get_module_dir(module_name), os.path.join(install_dir, module_name)) + shutil.copytree(self.get_module_dir(module_name), Path(install_dir, module_name)) # Switch back to the tip of the branch self.checkout_branch() diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index f99c91913a..b5cd2a610d 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -25,42 +25,49 @@ def param_check(self, module): if not self.has_valid_directory(): raise UserWarning() - if module is not None and module not in self.modules_json.get_all_modules().get(self.modules_repo.fullname, {}): - raise UserWarning(f"Module '{Path(self.modules_repo.fullname, module)}' does not exist in the pipeline") + modules = self.modules_json.get_all_modules()[self.modules_repo.remote_url] + module_names = [module for _, module in modules] + + if module is not None and module not in module_names: + module_dir = [dir for dir, m in modules if m == module][0] + raise UserWarning(f"Module '{Path('modules', module_dir, module)}' does not exist in the pipeline") def patch(self, module=None): self.modules_json.check_up_to_date() self.param_check(module) + modules = self.modules_json.get_all_modules()[self.modules_repo.remote_url] if module is None: + choices = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] module = questionary.autocomplete( "Tool:", - self.modules_json.get_all_modules()[self.modules_repo.fullname], + choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - module_fullname = str(Path(self.modules_repo.fullname, module)) + module_dir = [dir for dir, m in modules if m == module][0] + module_fullname = str(Path("modules", module_dir, module)) # Verify that the module has an entry is the modules.json file - if not self.modules_json.module_present(module, self.modules_repo.fullname): + if not self.modules_json.module_present(module, self.modules_repo.remote_url, module_dir): raise UserWarning( f"The '{module_fullname}' module does not have an entry in the 'modules.json' file. Cannot compute patch" ) - module_version = self.modules_json.get_module_version(module, self.modules_repo.remote_url) + module_version = self.modules_json.get_module_version(module, self.modules_repo.remote_url, module_dir) if module_version is None: raise UserWarning( f"The '{module_fullname}' module does not have a valid version in the 'modules.json' file. Cannot compute patch" ) # Get the module branch and reset it in the ModulesRepo object - module_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url) + module_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url, module_dir) if module_branch != self.modules_repo.branch: self.modules_repo.setup_branch(module_branch) # Set the diff filename based on the module name patch_filename = f"{module.replace('/', '-')}.diff" - module_relpath = Path("modules", self.modules_repo.fullname, module) + module_relpath = Path("modules", module_dir, module) patch_relpath = Path(module_relpath, patch_filename) - module_dir = Path(self.dir, module_relpath) + module_current_dir = Path(self.dir, module_relpath) patch_path = Path(self.dir, patch_relpath) if patch_path.exists(): @@ -89,25 +96,27 @@ def patch(self, module=None): module, self.modules_repo.fullname, module_install_dir, - module_dir, + module_current_dir, for_git=False, dsp_from_dir=module_relpath, dsp_to_dir=module_relpath, ) + log.debug(f"Patch file wrote to a temporary directory {patch_temp_path}") except UserWarning: raise UserWarning(f"Module '{module_fullname}' is unchanged. No patch to compute") # Write changes to modules.json - self.modules_json.add_patch_entry(module, self.modules_repo.fullname, patch_relpath) + self.modules_json.add_patch_entry(module, self.modules_repo.remote_url, module_dir, patch_relpath) + log.debug(f"Wrote patch path for module {module} to modules.json") # Show the changes made to the module ModulesDiffer.print_diff( module, self.modules_repo.fullname, module_install_dir, - module_dir, - dsp_from_dir=module_dir, - dsp_to_dir=module_dir, + module_current_dir, + dsp_from_dir=module_current_dir, + dsp_to_dir=module_current_dir, ) # Finally move the created patch file to its final location diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 3a248bd647..f628108838 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -25,16 +25,16 @@ def remove(self, module): self.has_valid_directory() self.has_modules_file() - repo_name = self.modules_repo.fullname + repo_dir = self.modules_repo.fullname if module is None: module = questionary.autocomplete( "Tool name:", - choices=self.modules_from_repo(repo_name), + choices=self.modules_from_repo(repo_dir), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() # Get the module directory - module_dir = Path(self.dir, "modules", repo_name, module) + module_dir = Path(self.dir, "modules", repo_dir, module) # Load the modules.json file modules_json = ModulesJson(self.dir) @@ -44,15 +44,15 @@ def remove(self, module): if not module_dir.exists(): log.error(f"Module directory does not exist: '{module_dir}'") - if modules_json.module_present(module, repo_name): + if modules_json.module_present(module, repo_dir): log.error(f"Found entry for '{module}' in 'modules.json'. Removing...") - modules_json.remove_entry(module, repo_name) + modules_json.remove_entry(module, self.modules_repo.remote_url, repo_dir) return False log.info(f"Removing {module}") # Remove entry from modules.json - modules_json.remove_entry(module, repo_name) + modules_json.remove_entry(module, self.modules_repo.remote_url, repo_dir) # Remove the module return self.clear_module_dir(module_name=module, module_dir=module_dir) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index cdcbe35b88..7c00dcd095 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -130,11 +130,13 @@ def update(self, module=None): exit_value = True all_patches_successful = True for modules_repo, module, sha, patch_relpath in modules_info: - module_fullname = str(Path(modules_repo.fullname, module)) + module_fullname = str(Path("modules", modules_repo.fullname, module)) # Are we updating the files in place or not? dry_run = self.show_diff or self.save_diff_fn - current_version = self.modules_json.get_module_version(module, modules_repo.remote_url) + current_version = self.modules_json.get_module_version( + module, modules_repo.remote_url, modules_repo.fullname + ) # Set the temporary installation folder install_dir = Path(tempfile.mkdtemp()) @@ -261,19 +263,24 @@ def get_single_module_info(self, module): UserWarning: If the '.nf-core.yml' entry is not valid. """ # Check if there are any modules installed from the repo - repo_name = self.modules_repo.fullname - if repo_name not in self.modules_json.get_all_modules(): - raise LookupError(f"No modules installed from '{repo_name}'") + repo_url = self.modules_repo.remote_url + modules = self.modules_json.get_all_modules().get(repo_url) + choices = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] + if repo_url not in self.modules_json.get_all_modules(): + raise LookupError(f"No modules installed from '{repo_url}'") if module is None: + choices = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] module = questionary.autocomplete( "Tool name:", - choices=self.modules_json.get_all_modules()[repo_name], + choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() + install_dir = [dir for dir, m in modules if module == m][0] + # Check if module is installed before trying to update - if module not in self.modules_json.get_all_modules()[repo_name]: + if module not in choices: raise LookupError(f"Module '{module}' is not installed in pipeline and could therefore not be updated") # Check that the supplied name is an available module @@ -303,11 +310,11 @@ def get_single_module_info(self, module): log.info(f"Updating module to ({sha})") # Check if the update branch is the same as the installation branch - current_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url) + current_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url, install_dir) new_branch = self.modules_repo.branch if current_branch != new_branch: log.warning( - f"You are trying to update the '{Path(self.modules_repo.fullname, module)}' module from " + f"You are trying to update the '{Path(install_dir, module)}' module from " f"the '{new_branch}' branch. This module was installed from the '{current_branch}'" ) switch = questionary.confirm(f"Do you want to update using the '{current_branch}' instead?").unsafe_ask() @@ -316,7 +323,7 @@ def get_single_module_info(self, module): self.modules_repo.setup_branch(current_branch) # If there is a patch file, get its filename - patch_fn = self.modules_json.get_patch_fn(module, self.modules_repo.fullname) + patch_fn = self.modules_json.get_patch_fn(module, self.modules_repo.remote_url, install_dir) return (self.modules_repo, module, sha, patch_fn) @@ -344,36 +351,72 @@ def get_all_modules_info(self, branch=None): # and check if they have an entry in the '.nf-core.yml' file for repo_name, modules in self.modules_json.get_all_modules().items(): if repo_name not in self.update_config or self.update_config[repo_name] is True: - modules_info[repo_name] = [ - (module, self.sha, self.modules_json.get_module_branch(module, repo_name)) for module in modules - ] + for module_dir, module in modules: + try: + modules_info[repo_name][module_dir].append( + (module, self.sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + (module, self.sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + ] elif isinstance(self.update_config[repo_name], dict): # If it is a dict, then there are entries for individual modules repo_config = self.update_config[repo_name] - modules_info[repo_name] = [] - for module in modules: + modules_info[repo_name] = {} + for module_dir, module in modules: if module not in repo_config or repo_config[module] is True: - modules_info[repo_name].append( - (module, self.sha, self.modules_json.get_module_branch(module, repo_name)) - ) + try: + modules_info[repo_name][module_dir].append( + ( + module, + self.sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + self.sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ] elif isinstance(repo_config[module], str): # If a string is given it is the commit SHA to which we should update to custom_sha = repo_config[module] - modules_info[repo_name].append( - (module, custom_sha, self.modules_json.get_module_branch(module, repo_name)) - ) + try: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ) + except KeyError: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ) if self.sha is not None: overridden_modules.append(module) elif repo_config[module] is False: # Otherwise the entry must be 'False' and we should ignore the module - skipped_modules.append(f"{repo_name}/{module}") + skipped_modules.append(f"{module_dir}/{module}") else: - raise UserWarning(f"Module '{module}' in '{repo_name}' has an invalid entry in '.nf-core.yml'") + raise UserWarning(f"Module '{module}' in '{module_dir}' has an invalid entry in '.nf-core.yml'") elif isinstance(self.update_config[repo_name], str): # If a string is given it is the commit SHA to which we should update to custom_sha = self.update_config[repo_name] - modules_info[repo_name] = [ - (module_name, custom_sha, self.modules_json.get_module_branch(module_name, repo_name)) + modules_info[repo_name]["nf-core"] = [ + ( + module_name, + custom_sha, + self.modules_json.get_module_branch(module_name, repo_name, "nf-core"), + ) for module_name in modules ] if self.sha is not None: @@ -406,33 +449,23 @@ def get_all_modules_info(self, branch=None): ) # Loop through modules_info and create on ModulesRepo object per remote and branch repos_and_branches = {} - for repo_name, mods in modules_info.items(): - for mod, sha, mod_branch in mods: - if branch is not None: - mod_branch = branch - if (repo_name, mod_branch) not in repos_and_branches: - repos_and_branches[(repo_name, mod_branch)] = [] - repos_and_branches[(repo_name, mod_branch)].append((mod, sha)) - - # Get the git urls from the modules.json - modules_info = ( - ( - repo_name, - self.modules_json.get_git_url(repo_name), - branch, - mods_shas, - ) - for (repo_name, branch), mods_shas in repos_and_branches.items() - ) + for repo_name, repo_content in modules_info.items(): + for module_dir, mods in repo_content.items(): + for mod, sha, mod_branch in mods: + if branch is not None: + mod_branch = branch + if (repo_name, mod_branch) not in repos_and_branches: + repos_and_branches[(repo_name, mod_branch)] = [] + repos_and_branches[(repo_name, mod_branch)].append((mod, sha)) # Create ModulesRepo objects repo_objs_mods = [] - for repo_name, repo_url, branch, mods_shas in modules_info: + for repo_url, branch, mods_shas in repos_and_branches.items(): try: modules_repo = ModulesRepo(remote_url=repo_url, branch=branch) except LookupError as e: log.warning(e) - log.info(f"Skipping modules in '{repo_name}'") + log.info(f"Skipping modules in '{repo_url}'") else: repo_objs_mods.append((modules_repo, mods_shas)) @@ -457,7 +490,8 @@ def get_all_modules_info(self, branch=None): # Add patch filenames to the modules that have them modules_info = [ - (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.fullname)) for repo, mod, sha in modules_info + (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.remote_url, repo.fullname)) + for repo, mod, sha in modules_info # TODO module_name, repo_url, install_dir ] return modules_info @@ -580,6 +614,8 @@ def try_apply_patch(self, module, repo_name, patch_relpath, module_dir, module_i shutil.copytree(temp_module_dir, module_install_dir) # Add the patch file to the modules.json file - self.modules_json.add_patch_entry(module, repo_name, patch_relpath, write_file=True) + self.modules_json.add_patch_entry( + module, repo_name, patch_relpath, write_file=True + ) # module_name, repo_url, install_dir return True diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index d6a991dadb..225e0e587f 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -3,18 +3,20 @@ "homePage": "https://github.com/{{ name }}", "repos": { "https://github.com/nf-core/modules.git": { - "modules/nf-core": { - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "fastqc": { - "branch": "master", - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "multiqc": { - "branch": "master", - "git_sha": "4b1d4bf401d4cf65ebc4f604bc8c8e7551109db3" + "modules": { + "nf-core": { + "custom/dumpsoftwareversions": { + "branch": "master", + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + }, + "fastqc": { + "branch": "master", + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + }, + "multiqc": { + "branch": "master", + "git_sha": "4b1d4bf401d4cf65ebc4f604bc8c8e7551109db3" + } } } } diff --git a/tests/modules/install.py b/tests/modules/install.py index 449db055c3..be17369e69 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -66,4 +66,6 @@ def test_modules_install_different_branch_succeed(self): # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + ) # TODO module, repo_url, install_dir diff --git a/tests/modules/patch.py b/tests/modules/patch.py index cb87c85d7a..0492bd4c07 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -69,7 +69,7 @@ def test_create_patch_no_change(self): # Check the 'modules.json' contains no patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) is None + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) is None # TODO module_name, repo_url, install_dir def test_create_patch_change(self): @@ -88,7 +88,7 @@ def test_create_patch_change(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -121,7 +121,7 @@ def test_create_patch_try_apply_successful(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -145,7 +145,7 @@ def test_create_patch_try_apply_successful(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -187,7 +187,7 @@ def test_create_patch_try_apply_failed(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -224,7 +224,7 @@ def test_create_patch_update_success(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -239,9 +239,11 @@ def test_create_patch_update_success(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir "modules", REPO_NAME, BISMARK_ALIGN, patch_fn - ), modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) + ), modules_json_obj.get_patch_fn( # TODO module_name, repo_url, install_dir + BISMARK_ALIGN, REPO_NAME + ) # TODO module_name, repo_url, install_dir # Check that the correct lines are in the patch file with open(module_path / patch_fn, "r") as fh: @@ -280,7 +282,7 @@ def test_create_patch_update_fail(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) diff --git a/tests/modules/update.py b/tests/modules/update.py index 8610ffd7f5..30ec720098 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -201,7 +201,9 @@ def test_update_different_branch_single_module(self): # Verify that the branch entry was updated correctly modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + ) # TODO module, repo_url, install_dir assert modules_json.get_module_version("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA @@ -223,10 +225,14 @@ def test_update_different_branch_mixed_modules_main(self): modules_json = ModulesJson(self.pipeline_dir) # Verify that the branch entry was updated correctly - assert modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + ) # TODO module, repo_url, install_dir assert modules_json.get_module_version("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA # MultiQC is present in both branches but should've been updated using the 'main' branch - assert modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_DEFAULT_BRANCH + assert ( + modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_DEFAULT_BRANCH + ) # TODO module, repo_url, install_dir def test_update_different_branch_mix_modules_branch_test(self): @@ -240,7 +246,9 @@ def test_update_different_branch_mix_modules_branch_test(self): update_obj.update() modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH + ) # TODO module, repo_url, install_dir assert modules_json.get_module_version("multiqc", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA From afd436a936f6432a7499b11d22c5be7a32f41025 Mon Sep 17 00:00:00 2001 From: Emilio Garcia Date: Tue, 27 Sep 2022 15:28:04 +0200 Subject: [PATCH 156/854] Update README.md Updating bioconda link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32d0d8f557..6ad2846b23 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ For documentation of the internal Python functions, please refer to the [Tools P You can install `nf-core/tools` from [bioconda](https://bioconda.github.io/recipes/nf-core/README.html). First, install conda and configure the channels to use bioconda -(see the [bioconda documentation](https://bioconda.github.io/user/install.html)). +(see the [bioconda documentation](https://bioconda.github.io/index.html#usage)). Then, just run the conda installation command: ```bash From 3e89a9e146706e3e5228d7da54b712ac9ef92664 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Sat, 1 Oct 2022 16:31:09 +0200 Subject: [PATCH 157/854] fix template --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 83d49a327b..51baff1cc2 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -91,7 +91,7 @@ workflow {{ short_name|upper }} { workflow_summary = Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.paramsSummaryMultiqc(workflow, summary_params) ch_workflow_summary = Channel.value(workflow_summary) - methods_description = WorkflowTestpipeline.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) + methods_description = Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) ch_methods_description = Channel.value(methods_description) ch_multiqc_files = Channel.empty() From f8c0cef932cf33802d8b4d43e3e190a364b0344f Mon Sep 17 00:00:00 2001 From: ggabernet Date: Sat, 1 Oct 2022 17:01:08 +0200 Subject: [PATCH 158/854] fix multiqc not run --- nf_core/pipeline-template/workflows/pipeline.nf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 51baff1cc2..74ed67a3bc 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -23,9 +23,9 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? file( params.multiqc_config, checkIfExists: true ) : Channel.empty() -ch_multiqc_logo = params.multiqc_logo ? file( params.multiqc_logo, checkIfExists: true ) : Channel.empty() +ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) +ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() +ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* @@ -102,9 +102,9 @@ workflow {{ short_name|upper }} { MULTIQC ( ch_multiqc_files.collect(), - ch_multiqc_config, - ch_multiqc_custom_config, - ch_multiqc_logo + ch_multiqc_config.collect().ifEmpty([]), + ch_multiqc_custom_config.collect().ifEmpty([]), + ch_multiqc_logo.collect().ifEmpty([]) ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) From cd5a32b5b3de614ba524c0397b4c6f0927e068ee Mon Sep 17 00:00:00 2001 From: ggabernet Date: Sat, 1 Oct 2022 19:32:44 +0200 Subject: [PATCH 159/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b1c0f7854..5b238a419f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Update MultiQC module, update supplying MultiQC default and custom config and logo files to module - Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) - Fix template spacing modified by JINJA ([#1830](https://github.com/nf-core/tools/pull/1830)) +- Fix MultiQC execution on template [#1855](https://github.com/nf-core/tools/pull/1855) ### Linting From 4cc55e5f8ecb9604a9a4826093f15e7194a0f02f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Sat, 1 Oct 2022 20:38:47 +0200 Subject: [PATCH 160/854] fix some pytests --- nf_core/modules/info.py | 2 +- nf_core/modules/install.py | 10 +- nf_core/modules/lint/__init__.py | 4 +- nf_core/modules/lint/main_nf.py | 2 +- nf_core/modules/lint/meta_yml.py | 2 +- nf_core/modules/lint/module_changes.py | 2 +- nf_core/modules/lint/module_patch.py | 2 +- nf_core/modules/lint/module_version.py | 2 +- nf_core/modules/list.py | 4 +- nf_core/modules/module_test.py | 2 +- nf_core/modules/module_utils.py | 24 +++ nf_core/modules/modules_command.py | 2 +- nf_core/modules/modules_differ.py | 22 +-- nf_core/modules/modules_json.py | 6 +- nf_core/modules/modules_repo.py | 30 ++-- nf_core/modules/nfcore_module.py | 4 +- nf_core/modules/patch.py | 4 +- nf_core/modules/remove.py | 9 +- nf_core/modules/update.py | 211 +++++++++++++++---------- tests/modules/update.py | 93 ++++++----- tests/test_modules.py | 4 +- tests/utils.py | 8 +- 22 files changed, 262 insertions(+), 187 deletions(-) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index dfc4431df3..6e4f0b2d3a 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -136,7 +136,7 @@ def get_local_yaml(self): if self.repo_type == "pipeline": # Try to find and load the meta.yml file - repo_name = self.modules_repo.fullname + repo_name = self.modules_repo.repo_path module_base_path = os.path.join(self.dir, "modules") # Check that we have any modules installed from this repo modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 54f0a94f20..392d177392 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -48,7 +48,7 @@ def install(self, module): # Verify that the provided SHA exists in the repo if self.sha: if not self.modules_repo.sha_exists_on_branch(self.sha): - log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.fullname}'") + log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") return False if module is None: @@ -72,11 +72,11 @@ def install(self, module): return False current_version = modules_json.get_module_version( - module, self.modules_repo.remote_url, self.modules_repo.fullname + module, self.modules_repo.remote_url, self.modules_repo.repo_path ) # Set the install folder based on the repository name - install_folder = os.path.join(self.dir, "modules", self.modules_repo.fullname) + install_folder = os.path.join(self.dir, "modules", self.modules_repo.repo_path) # Compute the module directory module_dir = os.path.join(install_folder, module) @@ -86,7 +86,7 @@ def install(self, module): log.error("Module is already installed.") repo_flag = ( - "" if self.modules_repo.fullname == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " + "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " ) branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " @@ -112,7 +112,7 @@ def install(self, module): version = self.modules_repo.get_latest_module_version(module) if self.force: - log.info(f"Removing installed version of '{self.modules_repo.fullname}/{module}'") + log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") self.clear_module_dir(module, module_dir) log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'") diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 83152ac222..91961fde60 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -88,9 +88,9 @@ def __init__( if self.modules_repo.remote_url in all_pipeline_modules: module_dir = Path(self.dir, "modules", "nf-core") self.all_remote_modules = [ - NFCoreModule(m, self.modules_repo.fullname, module_dir / m, self.repo_type, Path(self.dir)) + NFCoreModule(m[1], self.modules_repo.remote_url, module_dir / m[1], self.repo_type, Path(self.dir)) for m in all_pipeline_modules[self.modules_repo.remote_url] - ] + ] # m = (module_dir, module_name) if not self.all_remote_modules: raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.") local_module_dir = Path(self.dir, "modules", "local") diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4b5327020f..b642825db4 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -46,7 +46,7 @@ def main_nf(module_lint_object, module, fix_version, progress_bar): if module.is_patched: lines = ModulesDiffer.try_apply_patch( module.module_name, - module_lint_object.modules_repo.fullname, + module_lint_object.modules_repo.repo_path, module.patch_path, Path(module.module_dir).relative_to(module.base_dir), reverse=True, diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 146796fde2..77df16deb2 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -28,7 +28,7 @@ def meta_yml(module_lint_object, module): if module.is_patched: lines = ModulesDiffer.try_apply_patch( module.module_name, - module_lint_object.modules_repo.fullname, + module_lint_object.modules_repo.repo_path, module.patch_path, Path(module.module_dir).relative_to(module.base_dir), reverse=True, diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index 74e5df064e..c2f1c2e1dd 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -29,7 +29,7 @@ def module_changes(module_lint_object, module): shutil.copytree(module.module_dir, tempdir) try: new_lines = ModulesDiffer.try_apply_patch( - module.module_name, module_lint_object.modules_repo.fullname, module.patch_path, tempdir, reverse=True + module.module_name, module_lint_object.modules_repo.repo_path, module.patch_path, tempdir, reverse=True ) for file, lines in new_lines.items(): with open(tempdir / file, "w") as fh: diff --git a/nf_core/modules/lint/module_patch.py b/nf_core/modules/lint/module_patch.py index e6656136d1..6d91b44816 100644 --- a/nf_core/modules/lint/module_patch.py +++ b/nf_core/modules/lint/module_patch.py @@ -163,7 +163,7 @@ def patch_reversible(module_lint_object, module, patch_path): try: ModulesDiffer.try_apply_patch( module.module_name, - module_lint_object.modules_repo.fullname, + module_lint_object.modules_repo.repo_path, patch_path, Path(module.module_dir).relative_to(module.base_dir), reverse=True, diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index d2b7b1524d..9077cc5371 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -26,7 +26,7 @@ def module_version(module_lint_object, module): # Verify that a git_sha exists in the `modules.json` file for this module version = module_lint_object.modules_json.get_module_version( - module.module_name, module_lint_object.modules_repo.remote_url, module_lint_object.modules_repo.fullname + module.module_name, module_lint_object.modules_repo.remote_url, module_lint_object.modules_repo.repo_path ) if version is None: module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path)) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index 046efb1d7e..81e7929058 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -49,7 +49,7 @@ def pattern_msg(keywords): # Nothing found if len(modules) == 0: log.info( - f"No available modules found in {self.modules_repo.fullname} ({self.modules_repo.branch})" + f"No available modules found in {self.modules_repo.remote_url} ({self.modules_repo.branch})" f"{pattern_msg(keywords)}" ) return "" @@ -129,7 +129,7 @@ def pattern_msg(keywords): if self.remote: log.info( - f"Modules available from {self.modules_repo.fullname} ({self.modules_repo.branch})" + f"Modules available from {self.modules_repo.remote_url} ({self.modules_repo.branch})" f"{pattern_msg(keywords)}:\n" ) else: diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 339f5a1eb7..107b0c9545 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -83,7 +83,7 @@ def _check_inputs(self): else: modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - installed_modules = modules_json.get_all_modules().get(self.modules_repo.fullname) + installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url) # Get the tool name if not specified if self.module_name is None: diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 23315cc7f1..49b58bec64 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -47,6 +47,30 @@ def path_from_remote(remote_url): return path +def repo_full_name_from_remote(remote_url): + """ + Extracts the path from the remote URL + See https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS for the possible URL patterns + """ + # Check whether we have a https or ssh url + if remote_url.startswith("https"): + path = urllib.parse.urlparse(remote_url) + path = path.path + # Remove the intial '/' + path = path[1:] + # Remove extension + path = os.path.splitext(path)[0] + else: + # Remove the initial `git@`` + path = remote_url.split("@") + path = path[-1] if len(path) > 1 else path[0] + path = urllib.parse.urlparse(path) + path = path.path + # Remove extension + path = os.path.splitext(path)[0] + return path + + def get_installed_modules(dir, repo_type="modules"): """ Make a list of all modules installed in this repository diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 9c19799bff..7e622b18a2 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -117,7 +117,7 @@ def install_module_files(self, module_name, module_version, modules_repo, instal Args: module_name (str): The name of the module - module_versioN (str): Git SHA for the version of the module to be installed + module_version (str): Git SHA for the version of the module to be installed modules_repo (ModulesRepo): A correctly configured ModulesRepo object install_dir (str): The path to where the module should be installed (should be the 'modules/' dir of the pipeline) diff --git a/nf_core/modules/modules_differ.py b/nf_core/modules/modules_differ.py index cc919c722a..efce3868e5 100644 --- a/nf_core/modules/modules_differ.py +++ b/nf_core/modules/modules_differ.py @@ -124,7 +124,7 @@ def get_module_diffs(from_dir, to_dir, for_git=True, dsp_from_dir=None, dsp_to_d def write_diff_file( diff_path, module, - repo_name, + repo_path, from_dir, to_dir, current_version=None, @@ -140,7 +140,7 @@ def write_diff_file( Args: diff_path (str | Path): The path to the file that should be appended module (str): The module name - repo_name (str): The name of the repo where the module resides + repo_path (str): The name of the repo where the module resides from_dir (str | Path): The directory containing the old module files to_dir (str | Path): The directory containing the new module files diffs (dict[str, (ModulesDiffer.DiffEnum, str)]): A dictionary containing @@ -167,12 +167,12 @@ def write_diff_file( with open(diff_path, file_action) as fh: if current_version is not None and new_version is not None: fh.write( - f"Changes in module '{Path(repo_name, module)}' between" + f"Changes in module '{Path(repo_path, module)}' between" f" ({current_version}) and" f" ({new_version})\n" ) else: - fh.write(f"Changes in module '{Path(repo_name, module)}'\n") + fh.write(f"Changes in module '{Path(repo_path, module)}'\n") for _, (diff_status, diff) in diffs.items(): if diff_status != ModulesDiffer.DiffEnum.UNCHANGED: @@ -219,14 +219,14 @@ def append_modules_json_diff(diff_path, old_modules_json, new_modules_json, modu @staticmethod def print_diff( - module, repo_name, from_dir, to_dir, current_version=None, new_version=None, dsp_from_dir=None, dsp_to_dir=None + module, repo_path, from_dir, to_dir, current_version=None, new_version=None, dsp_from_dir=None, dsp_to_dir=None ): """ Prints the diffs between two module versions to the terminal Args: module (str): The module name - repo_name (str): The name of the repo where the module resides + repo_path (str): The name of the repo where the module resides from_dir (str | Path): The directory containing the old module files to_dir (str | Path): The directory containing the new module files module_dir (str): The path to the current installation of the module @@ -246,10 +246,10 @@ def print_diff( console = Console(force_terminal=nf_core.utils.rich_force_colors()) if current_version is not None and new_version is not None: log.info( - f"Changes in module '{Path(repo_name, module)}' between" f" ({current_version}) and" f" ({new_version})" + f"Changes in module '{Path(repo_path, module)}' between" f" ({current_version}) and" f" ({new_version})" ) else: - log.info(f"Changes in module '{Path(repo_name, module)}'") + log.info(f"Changes in module '{Path(repo_path, module)}'") for file, (diff_status, diff) in diffs.items(): if diff_status == ModulesDiffer.DiffEnum.UNCHANGED: @@ -423,13 +423,13 @@ def try_apply_single_patch(file_lines, patch, reverse=False): return patched_new_lines @staticmethod - def try_apply_patch(module, repo_name, patch_path, module_dir, reverse=False): + def try_apply_patch(module, repo_path, patch_path, module_dir, reverse=False): """ Try applying a full patch file to a module Args: module (str): Name of the module - repo_name (str): Name of the repository where the module resides + repo_path (str): Name of the repository where the module resides patch_path (str): The absolute path to the patch file to be applied module_dir (Path): The directory containing the module @@ -440,7 +440,7 @@ def try_apply_patch(module, repo_name, patch_path, module_dir, reverse=False): Raises: LookupError: If the patch application fails in a file """ - module_relpath = Path("modules", repo_name, module) + module_relpath = Path("modules", repo_path, module) patches = ModulesDiffer.per_file_patch(patch_path) new_files = {} for file, patch in patches.items(): diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 636f3491ed..32b6e447cf 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -559,7 +559,7 @@ def update(self, modules_repo, module_name, module_version, write_file=True): """ if self.modules_json is None: self.load() - repo_name = modules_repo.fullname + repo_name = modules_repo.repo_path remote_url = modules_repo.remote_url branch = modules_repo.branch if remote_url not in self.modules_json["repos"]: @@ -745,8 +745,8 @@ def get_all_modules(self): Retrieves all pipeline modules that are reported in the modules.json Returns: - (dict[str, [str]]): Dictionary indexed with the repo names, with a - list of modules as values + (dict[str, [(str, str)]]): Dictionary indexed with the repo urls, with a + list of tuples (module_dir, module) as values """ if self.modules_json is None: self.load() diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index e21048077f..4e88c51e13 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -2,6 +2,7 @@ import logging import os import shutil +from importlib.resources import path from pathlib import Path import git @@ -17,7 +18,7 @@ # Constants for the nf-core/modules repo used throughout the module files NF_CORE_MODULES_NAME = "nf-core" NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" -NF_CORE_MODULES_DEFAULT_BRANCH = "master" +NF_CORE_MODULES_DEFAULT_BRANCH = "restructure" class RemoteProgressbar(git.RemoteProgress): @@ -125,16 +126,17 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url - self.fullname = nf_core.modules.module_utils.path_from_remote(self.remote_url) + self.repo_path = nf_core.modules.module_utils.path_from_remote(self.remote_url) + self.fullname = nf_core.modules.module_utils.repo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress) # Verify that the repo seems to be correctly configured - if self.fullname != NF_CORE_MODULES_NAME or self.branch: + if self.repo_path != NF_CORE_MODULES_NAME or self.branch: self.verify_branch() # Convenience variable - self.modules_dir = os.path.join(self.local_repo_dir, "modules") + self.modules_dir = os.path.join(self.local_repo_dir, "modules", self.repo_path) self.avail_module_names = None @@ -149,7 +151,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True): branch (str): name of branch to use Sets self.repo """ - self.local_repo_dir = os.path.join(NFCORE_DIR, self.fullname, "modules") + self.local_repo_dir = os.path.join(NFCORE_DIR, self.fullname) if not os.path.exists(self.local_repo_dir): try: pbar = rich.progress.Progress( @@ -210,8 +212,8 @@ def setup_branch(self, branch): """ if branch is None: # Don't bother fetching default branch if we're using nf-core - if self.fullname == NF_CORE_MODULES_NAME: - self.branch = "master" + if self.remote_url == NF_CORE_MODULES_REMOTE: + self.branch = "restructure" else: self.branch = self.get_default_branch() else: @@ -235,7 +237,7 @@ def branch_exists(self): try: self.checkout_branch() except GitCommandError: - raise LookupError(f"Branch '{self.branch}' not found in '{self.fullname}'") + raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") def verify_branch(self): """ @@ -243,7 +245,7 @@ def verify_branch(self): """ dir_names = os.listdir(self.local_repo_dir) if "modules" not in dir_names: - err_str = f"Repository '{self.fullname}' ({self.branch}) does not contain the 'modules/' directory" + err_str = f"Repository '{self.remote_url}' ({self.branch}) does not contain the 'modules/' directory" if "software" in dir_names: err_str += ( ".\nAs of nf-core/tools version 2.0, the 'software/' directory should be renamed to 'modules/'" @@ -308,8 +310,8 @@ def install_module(self, module_name, install_dir, commit): return False # Check if the module exists in the branch - if not self.module_exists(module_name, checkout=False): - log.error(f"The requested module does not exists in the '{self.branch}' of {self.fullname}'") + if not self.module_exists(module_name): + log.error(f"The requested module does not exists in the '{self.branch}' of {self.remote_url}'") return False # Copy the files from the repo to the install folder @@ -398,7 +400,7 @@ def get_commit_info(self, sha): date_obj = commit.committed_datetime date = str(date_obj.date()) return message, date - raise LookupError(f"Commit '{sha}' not found in the '{self.fullname}'") + raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'") def get_avail_modules(self, checkout=True): """ @@ -411,6 +413,10 @@ def get_avail_modules(self, checkout=True): if checkout: self.checkout_branch() # Module directories are characterized by having a 'main.nf' file + print(self.modules_dir) + print(os.path.exists(Path(self.modules_dir))) + print(os.listdir(self.modules_dir)) + print(list(os.walk(self.modules_dir))) avail_module_names = [ os.path.relpath(dirpath, start=self.modules_dir) for dirpath, _, file_names in os.walk(self.modules_dir) diff --git a/nf_core/modules/nfcore_module.py b/nf_core/modules/nfcore_module.py index 62b77ab35d..9c622b2bff 100644 --- a/nf_core/modules/nfcore_module.py +++ b/nf_core/modules/nfcore_module.py @@ -10,7 +10,7 @@ class NFCoreModule(object): Includes functionality for linting """ - def __init__(self, module_name, repo_name, module_dir, repo_type, base_dir, nf_core_module=True): + def __init__(self, module_name, repo_url, module_dir, repo_type, base_dir, nf_core_module=True): """ Initialize the object @@ -24,7 +24,7 @@ def __init__(self, module_name, repo_name, module_dir, repo_type, base_dir, nf_c nf-core or local module """ self.module_name = module_name - self.repo_name = repo_name + self.repo_url = repo_url self.module_dir = module_dir self.repo_type = repo_type self.base_dir = base_dir diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index b5cd2a610d..670d7ec108 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -94,7 +94,7 @@ def patch(self, module=None): ModulesDiffer.write_diff_file( patch_temp_path, module, - self.modules_repo.fullname, + self.modules_repo.repo_path, module_install_dir, module_current_dir, for_git=False, @@ -112,7 +112,7 @@ def patch(self, module=None): # Show the changes made to the module ModulesDiffer.print_diff( module, - self.modules_repo.fullname, + self.modules_repo.repo_path, module_install_dir, module_current_dir, dsp_from_dir=module_current_dir, diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index f628108838..dd872da779 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -26,6 +26,7 @@ def remove(self, module): self.has_modules_file() repo_dir = self.modules_repo.fullname + repo_path = self.modules_repo.repo_path if module is None: module = questionary.autocomplete( "Tool name:", @@ -34,7 +35,7 @@ def remove(self, module): ).unsafe_ask() # Get the module directory - module_dir = Path(self.dir, "modules", repo_dir, module) + module_dir = Path(self.dir, "modules", repo_path, module) # Load the modules.json file modules_json = ModulesJson(self.dir) @@ -44,15 +45,15 @@ def remove(self, module): if not module_dir.exists(): log.error(f"Module directory does not exist: '{module_dir}'") - if modules_json.module_present(module, repo_dir): + if modules_json.module_present(module, self.modules_repo.remote_url): log.error(f"Found entry for '{module}' in 'modules.json'. Removing...") - modules_json.remove_entry(module, self.modules_repo.remote_url, repo_dir) + modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) return False log.info(f"Removing {module}") # Remove entry from modules.json - modules_json.remove_entry(module, self.modules_repo.remote_url, repo_dir) + modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) # Remove the module return self.clear_module_dir(module_name=module, module_dir=module_dir) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 7c00dcd095..3b7b9090db 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -98,7 +98,7 @@ def update(self, module=None): # Verify that the provided SHA exists in the repo if self.sha is not None and not self.modules_repo.sha_exists_on_branch(self.sha): - log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.fullname}'") + log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") return False # Get the list of modules to update, and their version information @@ -130,20 +130,20 @@ def update(self, module=None): exit_value = True all_patches_successful = True for modules_repo, module, sha, patch_relpath in modules_info: - module_fullname = str(Path("modules", modules_repo.fullname, module)) + module_fullname = str(Path("modules", modules_repo.repo_path, module)) # Are we updating the files in place or not? dry_run = self.show_diff or self.save_diff_fn current_version = self.modules_json.get_module_version( - module, modules_repo.remote_url, modules_repo.fullname + module, modules_repo.remote_url, modules_repo.repo_path ) # Set the temporary installation folder - install_dir = Path(tempfile.mkdtemp()) - module_install_dir = install_dir / module + install_tmp_dir = Path(tempfile.mkdtemp()) + module_install_dir = install_tmp_dir / module # Compute the module directory - module_dir = os.path.join(self.dir, "modules", modules_repo.fullname, module) + module_dir = os.path.join(self.dir, "modules", modules_repo.repo_path, module) if sha is not None: version = sha @@ -163,13 +163,13 @@ def update(self, module=None): continue # Download module files - if not self.install_module_files(module, version, modules_repo, install_dir): + if not self.install_module_files(module, version, modules_repo, install_tmp_dir): exit_value = False continue if patch_relpath is not None: patch_successful = self.try_apply_patch( - module, modules_repo.fullname, patch_relpath, module_dir, module_install_dir + module, modules_repo.repo_path, patch_relpath, module_dir, module_install_dir ) if patch_successful: log.info(f"Module '{module_fullname}' patched successfully") @@ -189,7 +189,7 @@ def update(self, module=None): ModulesDiffer.write_diff_file( self.save_diff_fn, module, - modules_repo.fullname, + modules_repo.repo_path, module_dir, module_install_dir, current_version, @@ -201,7 +201,7 @@ def update(self, module=None): elif self.show_diff: ModulesDiffer.print_diff( module, - modules_repo.fullname, + modules_repo.repo_path, module_dir, module_install_dir, current_version, @@ -217,7 +217,7 @@ def update(self, module=None): if not dry_run: # Clear the module directory and move the installed files there - self.move_files_from_tmp_dir(module, module_dir, install_dir, modules_repo.fullname, version) + self.move_files_from_tmp_dir(module, install_tmp_dir, modules_repo.repo_path, version) # Update modules.json with newly installed module self.modules_json.update(modules_repo, module, version) else: @@ -270,13 +270,13 @@ def get_single_module_info(self, module): raise LookupError(f"No modules installed from '{repo_url}'") if module is None: - choices = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] module = questionary.autocomplete( "Tool name:", choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() + # Get module installation directory install_dir = [dir for dir, m in modules if module == m][0] # Check if module is installed before trying to update @@ -291,8 +291,9 @@ def get_single_module_info(self, module): ) sha = self.sha - if module in self.update_config.get(self.modules_repo.fullname, {}): - config_entry = self.update_config[self.modules_repo.fullname].get(module) + if module in self.update_config.get(self.modules_repo.remote_url, {}).get(install_dir, {}): + # If the module to update is in .nf-core.yml config file + config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(module) if config_entry is not None and config_entry is not True: if config_entry is False: raise UserWarning("Module's update entry in '.nf-core.yml' is set to False") @@ -351,6 +352,8 @@ def get_all_modules_info(self, branch=None): # and check if they have an entry in the '.nf-core.yml' file for repo_name, modules in self.modules_json.get_all_modules().items(): if repo_name not in self.update_config or self.update_config[repo_name] is True: + # There aren't restrictions for the repository in .nf-core.yml file + modules_info[repo_name] = {} for module_dir, module in modules: try: modules_info[repo_name][module_dir].append( @@ -361,64 +364,99 @@ def get_all_modules_info(self, branch=None): (module, self.sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) ] elif isinstance(self.update_config[repo_name], dict): - # If it is a dict, then there are entries for individual modules - repo_config = self.update_config[repo_name] - modules_info[repo_name] = {} - for module_dir, module in modules: - if module not in repo_config or repo_config[module] is True: - try: - modules_info[repo_name][module_dir].append( - ( - module, - self.sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), - ) - ) - except KeyError: - modules_info[repo_name][module_dir] = [ - ( - module, - self.sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), - ) - ] - elif isinstance(repo_config[module], str): + # If it is a dict, then there are entries for individual modules or module directories + for module_dir in set([dir for dir, _ in modules]): + if isinstance(self.update_config[repo_name][module_dir], str): # If a string is given it is the commit SHA to which we should update to - custom_sha = repo_config[module] - try: - modules_info[repo_name][module_dir].append( - ( - module, - custom_sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), - ) - ) - except KeyError: - modules_info[repo_name][module_dir].append( - ( - module, - custom_sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), - ) - ) + custom_sha = self.update_config[repo_name][module_dir] + modules_info[repo_name] = {} + for dir, module in modules: + if module_dir == dir: + try: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + custom_sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ] if self.sha is not None: - overridden_modules.append(module) - elif repo_config[module] is False: - # Otherwise the entry must be 'False' and we should ignore the module - skipped_modules.append(f"{module_dir}/{module}") - else: - raise UserWarning(f"Module '{module}' in '{module_dir}' has an invalid entry in '.nf-core.yml'") + overridden_repos.append(repo_name) + elif self.update_config[repo_name][module_dir] is False: + for dir, module in modules: + if dir == module_dir: + skipped_modules.append(f"{module_dir}/{module}") + elif isinstance(self.update_config[repo_name][module_dir], dict): + # If it's a dict, there are entries for individual modules + dir_config = self.update_config[repo_name][module_dir] + modules_info[repo_name] = {} + for module_dir, module in modules: + if module not in dir_config or dir_config[module] is True: + try: + modules_info[repo_name][module_dir].append( + ( + module, + self.sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + self.sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ] + elif isinstance(dir_config[module], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = dir_config[module] + try: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ) + except KeyError: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_module_branch(module, repo_name, module_dir), + ) + ) + if self.sha is not None: + overridden_modules.append(module) + elif dir_config[module] is False: + # Otherwise the entry must be 'False' and we should ignore the module + skipped_modules.append(f"{module_dir}/{module}") + else: + raise UserWarning( + f"Module '{module}' in '{module_dir}' has an invalid entry in '.nf-core.yml'" + ) elif isinstance(self.update_config[repo_name], str): # If a string is given it is the commit SHA to which we should update to custom_sha = self.update_config[repo_name] - modules_info[repo_name]["nf-core"] = [ - ( - module_name, - custom_sha, - self.modules_json.get_module_branch(module_name, repo_name, "nf-core"), - ) - for module_name in modules - ] + modules_info[repo_name] = {} + for module_dir, module in modules: + try: + modules_info[repo_name][module_dir].append( + (module, custom_sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + (module, custom_sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + ] if self.sha is not None: overridden_repos.append(repo_name) elif self.update_config[repo_name] is False: @@ -460,7 +498,7 @@ def get_all_modules_info(self, branch=None): # Create ModulesRepo objects repo_objs_mods = [] - for repo_url, branch, mods_shas in repos_and_branches.items(): + for (repo_url, branch), mods_shas in repos_and_branches.items(): try: modules_repo = ModulesRepo(remote_url=repo_url, branch=branch) except LookupError as e: @@ -478,11 +516,11 @@ def get_all_modules_info(self, branch=None): while i < len(modules_info): repo, module, sha = modules_info[i] if not repo.module_exists(module): - log.warning(f"Module '{module}' does not exist in '{repo.fullname}'. Skipping...") + log.warning(f"Module '{module}' does not exist in '{repo.remote_url}'. Skipping...") modules_info.pop(i) elif sha is not None and not repo.sha_exists_on_branch(sha): log.warning( - f"Git sha '{sha}' does not exists on the '{repo.branch}' of '{repo.fullname}'. Skipping module '{module}'" + f"Git sha '{sha}' does not exists on the '{repo.branch}' of '{repo.remote_url}'. Skipping module '{module}'" ) modules_info.pop(i) else: @@ -490,8 +528,8 @@ def get_all_modules_info(self, branch=None): # Add patch filenames to the modules that have them modules_info = [ - (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.remote_url, repo.fullname)) - for repo, mod, sha in modules_info # TODO module_name, repo_url, install_dir + (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.remote_url, repo.repo_path)) + for repo, mod, sha in modules_info ] return modules_info @@ -525,39 +563,38 @@ def setup_diff_file(self): # This guarantees that the file exists after calling the function self.save_diff_fn.touch() - def move_files_from_tmp_dir(self, module, module_dir, install_folder, repo_name, new_version): + def move_files_from_tmp_dir(self, module, install_folder, repo_path, new_version): """Move the files from the temporary to the installation directory. Args: module (str): The module name. - module_dir (str): The path to the module directory. install_folder [str]: The path to the temporary installation directory. - modules_repo (ModulesRepo): The ModulesRepo object from which the module was installed. + repo_path (str): The name of the directory where modules are installed new_version (str): The version of the module that was installed. """ temp_module_dir = os.path.join(install_folder, module) files = os.listdir(temp_module_dir) log.debug(f"Removing old version of module '{module}'") - self.clear_module_dir(module, module_dir) + self.clear_module_dir(module, repo_path) - os.makedirs(module_dir) + os.makedirs(repo_path) for file in files: path = os.path.join(temp_module_dir, file) if os.path.exists(path): - shutil.move(path, os.path.join(module_dir, file)) + shutil.move(path, os.path.join(repo_path, file)) - log.info(f"Updating '{repo_name}/{module}'") - log.debug(f"Updating module '{module}' to {new_version} from {repo_name}") + log.info(f"Updating '{repo_path}/{module}'") + log.debug(f"Updating module '{module}' to {new_version} from {repo_path}") - def try_apply_patch(self, module, repo_name, patch_relpath, module_dir, module_install_dir): + def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_install_dir): """ Try applying a patch file to the new module files Args: module (str): The name of the module - repo_name (str): The name of the repository where the module resides + repo_path (str): The name of the repository where the module resides patch_relpath (Path | str): The path to patch file in the pipeline module_dir (Path | str): The module directory in the pipeline module_install_dir (Path | str): The directory where the new module @@ -566,11 +603,11 @@ def try_apply_patch(self, module, repo_name, patch_relpath, module_dir, module_i Returns: (bool): Whether the patch application was successful """ - module_fullname = str(Path(repo_name, module)) + module_fullname = str(Path(repo_path, module)) log.info(f"Found patch for module '{module_fullname}'. Trying to apply it to new files") patch_path = Path(self.dir / patch_relpath) - module_relpath = Path("modules", repo_name, module) + module_relpath = Path("modules", repo_path, module) # Copy the installed files to a new temporary directory to save them for later use temp_dir = Path(tempfile.mkdtemp()) @@ -578,7 +615,7 @@ def try_apply_patch(self, module, repo_name, patch_relpath, module_dir, module_i shutil.copytree(module_install_dir, temp_module_dir) try: - new_files = ModulesDiffer.try_apply_patch(module, repo_name, patch_path, temp_module_dir) + new_files = ModulesDiffer.try_apply_patch(module, repo_path, patch_path, temp_module_dir) except LookupError: # Patch failed. Save the patch file by moving to the install dir shutil.move(patch_path, Path(module_install_dir, patch_path.relative_to(module_dir))) @@ -599,7 +636,7 @@ def try_apply_patch(self, module, repo_name, patch_relpath, module_dir, module_i ModulesDiffer.write_diff_file( Path(temp_module_dir, patch_path.relative_to(module_dir)), module, - repo_name, + repo_path, module_install_dir, temp_module_dir, file_action="w", @@ -615,7 +652,7 @@ def try_apply_patch(self, module, repo_name, patch_relpath, module_dir, module_i # Add the patch file to the modules.json file self.modules_json.add_patch_entry( - module, repo_name, patch_relpath, write_file=True + module, repo_path, patch_relpath, write_file=True ) # module_name, repo_url, install_dir return True diff --git a/tests/modules/update.py b/tests/modules/update.py index 30ec720098..5ab3917248 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -8,7 +8,7 @@ import nf_core.utils from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.update import ModuleUpdate from ..utils import ( @@ -56,7 +56,9 @@ def test_install_at_hash_and_update(self): mod_json = mod_json_obj.get_modules_json() # Get the up-to-date git_sha for the module from the ModulesRepo object correct_git_sha = update_obj.modules_repo.get_latest_module_version("trimgalore") - current_git_sha = mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"][ + "git_sha" + ] assert correct_git_sha == current_git_sha @@ -88,9 +90,9 @@ def test_update_all(self): mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() # Loop through all modules and check that they are updated (according to the modules.json file) - for mod in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]: + for mod in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: correct_git_sha = list(update_obj.modules_repo.get_module_git_log(mod, depth=1))[0]["git_sha"] - current_git_sha = mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][mod]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] assert correct_git_sha == current_git_sha @@ -100,7 +102,7 @@ def test_update_with_config_fixed_version(self): self.mods_install.install("trimgalore") # Fix the trimgalore version in the .nf-core.yml to an old version - update_config = {"nf-core/modules": {"trimgalore": OLD_TRIMGALORE_SHA}} + update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"trimgalore": OLD_TRIMGALORE_SHA}}} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: @@ -112,9 +114,12 @@ def test_update_with_config_fixed_version(self): # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"] - assert mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"] + assert ( + mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"]["git_sha"] + == OLD_TRIMGALORE_SHA + ) def test_update_with_config_dont_update(self): @@ -123,7 +128,7 @@ def test_update_with_config_dont_update(self): self.mods_install_old.install("trimgalore") # Set the trimgalore field to no update in the .nf-core.yml - update_config = {"nf-core/modules": {"trimgalore": False}} + update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"trimgalore": False}}} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: @@ -135,9 +140,12 @@ def test_update_with_config_dont_update(self): # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"] - assert mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"] + assert ( + mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"]["git_sha"] + == OLD_TRIMGALORE_SHA + ) def test_update_with_config_fix_all(self): @@ -145,7 +153,7 @@ def test_update_with_config_fix_all(self): self.mods_install.install("trimgalore") # Fix the version of all nf-core modules in the .nf-core.yml to an old version - update_config = {"nf-core/modules": OLD_TRIMGALORE_SHA} + update_config = {NF_CORE_MODULES_REMOTE: OLD_TRIMGALORE_SHA} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: @@ -157,9 +165,11 @@ def test_update_with_config_fix_all(self): # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - for module in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]: - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module] - assert mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module]["git_sha"] == OLD_TRIMGALORE_SHA + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"] + assert ( + mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"]["git_sha"] + == OLD_TRIMGALORE_SHA + ) def test_update_with_config_no_updates(self): @@ -168,7 +178,7 @@ def test_update_with_config_no_updates(self): old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Fix the version of all nf-core modules in the .nf-core.yml to an old version - update_config = {"nf-core/modules": False} + update_config = {NF_CORE_MODULES_REMOTE: False} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: @@ -178,13 +188,13 @@ def test_update_with_config_no_updates(self): update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) assert update_obj.update() is True - # Check that the git sha for trimgalore is correctly downgraded + # Check that the git sha for trimgalore is correctly downgraded and none of the modules has changed mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - for module in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]: - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module] + for module in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][module] assert ( - mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module]["git_sha"] - == old_mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][module]["git_sha"] + mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][module]["git_sha"] + == old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][module]["git_sha"] ) @@ -193,7 +203,8 @@ def test_update_different_branch_single_module(self): install_obj = ModuleInstall( self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA ) - install_obj.install("fastp") + installed = install_obj.install("fastp") + assert installed update_obj = ModuleUpdate( self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False ) @@ -201,10 +212,8 @@ def test_update_different_branch_single_module(self): # Verify that the branch entry was updated correctly modules_json = ModulesJson(self.pipeline_dir) - assert ( - modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH - ) # TODO module, repo_url, install_dir - assert modules_json.get_module_version("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA + assert modules_json.get_module_branch("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA def test_update_different_branch_mixed_modules_main(self): @@ -213,11 +222,11 @@ def test_update_different_branch_mixed_modules_main(self): install_obj = ModuleInstall( self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA ) - install_obj.install("fastp") + assert install_obj.install("fastp") # Install MultiQC from gitlab default branch - install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH) - install_obj.install("multiqc") + install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH, force=True) + assert install_obj.install("multiqc") # Force as the same module is installed from github nf-core modules repo # Try updating update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) @@ -225,31 +234,29 @@ def test_update_different_branch_mixed_modules_main(self): modules_json = ModulesJson(self.pipeline_dir) # Verify that the branch entry was updated correctly - assert ( - modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH - ) # TODO module, repo_url, install_dir - assert modules_json.get_module_version("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA + assert modules_json.get_module_branch("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA # MultiQC is present in both branches but should've been updated using the 'main' branch - assert ( - modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_DEFAULT_BRANCH - ) # TODO module, repo_url, install_dir + assert modules_json.get_module_branch("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_DEFAULT_BRANCH def test_update_different_branch_mix_modules_branch_test(self): """Try updating all modules where MultiQC is installed from branch-test branch""" # Install multiqc from the branch-test branch install_obj = ModuleInstall( - self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + self.pipeline_dir, + remote_url=GITLAB_URL, + branch=GITLAB_BRANCH_TEST_BRANCH, + sha=GITLAB_BRANCH_TEST_OLD_SHA, + force=True, ) - install_obj.install("multiqc") + install_obj.install("multiqc") # Force as the same module is installed from github nf-core modules repo update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) update_obj.update() modules_json = ModulesJson(self.pipeline_dir) - assert ( - modules_json.get_module_branch("multiqc", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH - ) # TODO module, repo_url, install_dir - assert modules_json.get_module_version("multiqc", GITLAB_URL) == GITLAB_BRANCH_TEST_NEW_SHA + assert modules_json.get_module_branch("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_module_version("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA def cmp_module(dir1, dir2): diff --git a/tests/test_modules.py b/tests/test_modules.py index cf8c1c82f6..e5cec06c64 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -72,8 +72,8 @@ def tearDown(self): def test_modulesrepo_class(self): """Initialise a modules repo object""" modrepo = nf_core.modules.ModulesRepo() - assert modrepo.fullname == "nf-core/modules" - assert modrepo.branch == "master" + assert modrepo.repo_path == "nf-core" + assert modrepo.branch == "restructure" ############################################ # Test of the individual modules commands. # diff --git a/tests/utils.py b/tests/utils.py index 164df96c83..850b151ed5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,12 +12,12 @@ OLD_TRIMGALORE_SHA = "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" -GITLAB_REPO = "nf-core/modules-test" -GITLAB_DEFAULT_BRANCH = "main" +GITLAB_REPO = "nf-core" +GITLAB_DEFAULT_BRANCH = "main-restructure" # Branch test stuff -GITLAB_BRANCH_TEST_BRANCH = "branch-tester" +GITLAB_BRANCH_TEST_BRANCH = "branch-tester-restructure" GITLAB_BRANCH_TEST_OLD_SHA = "eb4bc244de7eaef8e8ff0d451e4ca2e4b2c29821" -GITLAB_BRANCH_TEST_NEW_SHA = "e43448a2cc17d59e085c4d3f77489af5a4dcc26d" +GITLAB_BRANCH_TEST_NEW_SHA = "a4ac8e1abb05dcd583da18208f2ca951de2a205e" def with_temporary_folder(func): From 8366a772b33aea4e45f81d9dd14b6da1d5a2d166 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Sun, 2 Oct 2022 16:33:43 +0200 Subject: [PATCH 161/854] more tests fixed --- nf_core/__main__.py | 16 ++++++-- nf_core/modules/create.py | 2 +- nf_core/modules/install.py | 10 +++++ nf_core/modules/modules_json.py | 19 +++++---- nf_core/modules/modules_repo.py | 4 -- nf_core/modules/nfcore_module.py | 3 +- nf_core/modules/remove.py | 2 +- nf_core/modules/test_yml_builder.py | 3 +- nf_core/modules/update.py | 4 +- nf_core/pipeline-template/modules.json | 6 +-- tests/modules/bump_versions.py | 4 +- tests/modules/create.py | 8 ++-- tests/modules/create_test_yml.py | 14 ++++--- tests/modules/install.py | 6 +-- tests/modules/list.py | 5 ++- tests/modules/modules_json.py | 56 +++++++++++++------------- tests/modules/patch.py | 28 ++++++------- tests/modules/update.py | 4 +- tests/test_modules.py | 9 ++--- 19 files changed, 112 insertions(+), 91 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 39cd1390ab..0feac32391 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -398,7 +398,9 @@ def list(ctx): @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def remote(ctx, keywords, json): +@click.option("-u", "--url", help="Remote repository url to list from") +@click.option("-b", "--branch", help="Remote repository branch to list") +def remote(ctx, keywords, json, url, branch): """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -406,8 +408,8 @@ def remote(ctx, keywords, json): module_list = nf_core.modules.ModuleList( None, True, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_branch"] if url is None else url, + ctx.obj["modules_repo_branch"] if branch is None else branch, ctx.obj["modules_repo_no_pull"], ) stdout.print(module_list.list_modules(keywords, json)) @@ -661,7 +663,13 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): the required `test.yml` file based on the output files. """ try: - meta_builder = nf_core.modules.ModulesTestYmlBuilder(tool, run_tests, output, force, no_prompts) + meta_builder = nf_core.modules.ModulesTestYmlBuilder( + module_name=tool, + run_tests=run_tests, + test_yml_output_path=output, + force_overwrite=force, + no_prompts=no_prompts, + ) meta_builder.run() except (UserWarning, LookupError) as e: log.critical(e) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 81f0bc4429..e50d2b22fe 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -323,7 +323,7 @@ def get_module_dirs(self): ) # Set file paths - file_paths[os.path.join(self.default_modules_path, "main.nf")] = module_file + file_paths[os.path.join("modules", "main.nf")] = module_file if self.repo_type == "modules": software_dir = os.path.join(self.directory, self.default_modules_path, self.tool_dir) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 392d177392..616963bbe8 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -114,6 +114,16 @@ def install(self, module): if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") self.clear_module_dir(module, module_dir) + for repo_url, repo_content in modules_json.modules_json["repos"].items(): + for dir, dir_modules in repo_content["modules"].items(): + for name, _ in dir_modules.items(): + if name == module and dir == self.modules_repo.repo_path: + repo_to_remove = repo_url + log.info( + f"Removing module '{self.modules_repo.repo_path}/{module}' from repo '{repo_to_remove}' from modules.json" + ) + modules_json.remove_entry(module, repo_to_remove, self.modules_repo.repo_path) + break log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'") log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}") diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 32b6e447cf..397d6abcf3 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -60,21 +60,26 @@ def create(self): ( repo_url, [ - str(Path(module_name).relative_to(modules_dir / dir)) - for module_name, _, file_names in os.walk(modules_dir / dir) + str( + Path(module_name).relative_to( + modules_dir / nf_core.modules.module_utils.path_from_remote(repo_url) + ) + ) + for module_name, _, file_names in os.walk( + modules_dir / nf_core.modules.module_utils.path_from_remote(repo_url) + ) if "main.nf" in file_names ], - dir, + nf_core.modules.module_utils.path_from_remote(repo_url), ) - for dir, modules in repo_dict["modules"].items() - for repo_url, repo_dict in repos.items() + for repo_url in repos ] for repo_url, module_names, install_dir in sorted(repo_module_names): modules_json["repos"][repo_url] = {} modules_json["repos"][repo_url]["modules"] = {} modules_json["repos"][repo_url]["modules"][install_dir] = {} - modules_json["repos"][repo_url]["modules"] = self.determine_module_branches_and_shas( + modules_json["repos"][repo_url]["modules"][install_dir] = self.determine_module_branches_and_shas( install_dir, repo_url, module_names ) # write the modules.json file and assign it to the object @@ -107,7 +112,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): renamed_dirs = {} # Check if there are any untracked repositories dirs_not_covered = self.dir_tree_uncovered( - modules_dir, [Path(name) for url in repos for name in repos[url][modules_dir]] + modules_dir, [Path(nf_core.modules.module_utils.path_from_remote(url)) for url in repos] ) if len(dirs_not_covered) > 0: log.info("Found custom module repositories when creating 'modules.json'") diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 4e88c51e13..5551bdaa9d 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -413,10 +413,6 @@ def get_avail_modules(self, checkout=True): if checkout: self.checkout_branch() # Module directories are characterized by having a 'main.nf' file - print(self.modules_dir) - print(os.path.exists(Path(self.modules_dir))) - print(os.listdir(self.modules_dir)) - print(list(os.walk(self.modules_dir))) avail_module_names = [ os.path.relpath(dirpath, start=self.modules_dir) for dirpath, _, file_names in os.walk(self.modules_dir) diff --git a/nf_core/modules/nfcore_module.py b/nf_core/modules/nfcore_module.py index 9c622b2bff..67bfc6c1c4 100644 --- a/nf_core/modules/nfcore_module.py +++ b/nf_core/modules/nfcore_module.py @@ -43,7 +43,8 @@ def __init__(self, module_name, repo_url, module_dir, repo_type, base_dir, nf_co self.main_nf = self.module_dir / "main.nf" self.meta_yml = self.module_dir / "meta.yml" - self.test_dir = Path(self.base_dir, "tests", self.module_dir) + repo_dir = self.module_dir.parts[: self.module_dir.parts.index(self.module_name.split("/")[0])][-1] + self.test_dir = Path(self.base_dir, "tests", "modules", repo_dir, self.module_name) self.test_yml = self.test_dir / "test.yml" self.test_main_nf = self.test_dir / "main.nf" diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index dd872da779..22ce36d169 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -45,7 +45,7 @@ def remove(self, module): if not module_dir.exists(): log.error(f"Module directory does not exist: '{module_dir}'") - if modules_json.module_present(module, self.modules_repo.remote_url): + if modules_json.module_present(module, self.modules_repo.remote_url, repo_path): log.error(f"Found entry for '{module}' in 'modules.json'. Removing...") modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) return False diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 7cbdd94b3b..d8733803d8 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -35,12 +35,13 @@ class ModulesTestYmlBuilder(ModuleCommand): def __init__( self, module_name=None, + directory=".", run_tests=False, test_yml_output_path=None, force_overwrite=False, no_prompts=False, ): - super().__init__(".") + super().__init__(directory) self.module_name = module_name self.run_tests = run_tests self.test_yml_output_path = test_yml_output_path diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 3b7b9090db..e6b58e717f 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -652,7 +652,7 @@ def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_i # Add the patch file to the modules.json file self.modules_json.add_patch_entry( - module, repo_path, patch_relpath, write_file=True - ) # module_name, repo_url, install_dir + module, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=True + ) return True diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 225e0e587f..21644f7c60 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -6,15 +6,15 @@ "modules": { "nf-core": { "custom/dumpsoftwareversions": { - "branch": "master", + "branch": "restructure", "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" }, "fastqc": { - "branch": "master", + "branch": "restructure", "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" }, "multiqc": { - "branch": "master", + "branch": "restructure", "git_sha": "4b1d4bf401d4cf65ebc4f604bc8c8e7551109db3" } } diff --git a/tests/modules/bump_versions.py b/tests/modules/bump_versions.py index 388b6be424..e8b5803f00 100644 --- a/tests/modules/bump_versions.py +++ b/tests/modules/bump_versions.py @@ -10,7 +10,7 @@ def test_modules_bump_versions_single_module(self): """Test updating a single module""" # Change the bpipe/test version to an older version - main_nf_path = os.path.join(self.nfcore_modules, "modules", "bpipe", "test", "main.nf") + main_nf_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf") with open(main_nf_path, "r") as fh: content = fh.read() new_content = re.sub(r"bioconda::star=\d.\d.\d\D?", r"bioconda::star=2.6.1d", content) @@ -39,7 +39,7 @@ def test_modules_bump_versions_fail(self): def test_modules_bump_versions_fail_unknown_version(self): """Fail because of an unknown version""" # Change the bpipe/test version to an older version - main_nf_path = os.path.join(self.nfcore_modules, "modules", "bpipe", "test", "main.nf") + main_nf_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf") with open(main_nf_path, "r") as fh: content = fh.read() new_content = re.sub(r"bioconda::bpipe=\d.\d.\d\D?", r"bioconda::bpipe=xxx", content) diff --git a/tests/modules/create.py b/tests/modules/create.py index 6c1767b138..1ea2d66fa7 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -29,8 +29,8 @@ def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "fastqc", "@author", "process_low", False, False) module_create.create() - assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "fastqc", "main.nf")) - assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "fastqc", "main.nf")) + assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "fastqc", "main.nf")) + assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "fastqc", "main.nf")) def test_modules_create_nfcore_modules_subtool(self): @@ -39,5 +39,5 @@ def test_modules_create_nfcore_modules_subtool(self): self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) module_create.create() - assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "star", "index", "main.nf")) - assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "star", "index", "main.nf")) + assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "main.nf")) + assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "star", "index", "main.nf")) diff --git a/tests/modules/create_test_yml.py b/tests/modules/create_test_yml.py index dfb5fb5c6c..0d06339375 100644 --- a/tests/modules/create_test_yml.py +++ b/tests/modules/create_test_yml.py @@ -11,7 +11,7 @@ def test_modules_custom_yml_dumper(self, out_dir): """Try to create a yml file with the custom yml dumper""" yml_output_path = os.path.join(out_dir, "test.yml") - meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", False, "./", False, True) + meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) meta_builder.test_yml_output_path = yml_output_path meta_builder.tests = [{"testname": "myname"}] meta_builder.print_test_yml() @@ -21,7 +21,7 @@ def test_modules_custom_yml_dumper(self, out_dir): @with_temporary_folder def test_modules_test_file_dict(self, test_file_dir): """Creat dict of test files and create md5 sums""" - meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", False, "./", False, True) + meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") test_files = meta_builder.create_test_file_dict(test_file_dir) @@ -32,7 +32,7 @@ def test_modules_test_file_dict(self, test_file_dir): @with_temporary_folder def test_modules_create_test_yml_get_md5(self, test_file_dir): """Get md5 sums from a dummy output""" - meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", False, "./", False, True) + meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") test_files = meta_builder.get_md5_sums( @@ -43,8 +43,10 @@ def test_modules_create_test_yml_get_md5(self, test_file_dir): def test_modules_create_test_yml_entry_points(self): """Test extracting test entry points from a main.nf file""" - meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", False, "./", False, True) - meta_builder.module_test_main = os.path.join(self.nfcore_modules, "tests", "modules", "bpipe", "test", "main.nf") + meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", self.pipeline_dir, False, "./", False, True) + meta_builder.module_test_main = os.path.join( + self.nfcore_modules, "tests", "modules", "nf-core", "bpipe", "test", "main.nf" + ) meta_builder.scrape_workflow_entry_points() assert meta_builder.entry_points[0] == "test_bpipe_test" @@ -53,7 +55,7 @@ def test_modules_create_test_yml_check_inputs(self): """Test the check_inputs() function - raise UserWarning because test.yml exists""" cwd = os.getcwd() os.chdir(self.nfcore_modules) - meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", False, "./", False, True) + meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", self.pipeline_dir, False, "./", False, True) meta_builder.module_test_main = os.path.join(self.nfcore_modules, "tests", "modules", "bpipe", "test", "main.nf") with pytest.raises(UserWarning) as excinfo: meta_builder.check_inputs() diff --git a/tests/modules/install.py b/tests/modules/install.py index be17369e69..06c5173346 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -36,7 +36,7 @@ def test_modules_install_nomodule(self): def test_modules_install_trimgalore(self): """Test installing a module - TrimGalore!""" assert self.mods_install.install("trimgalore") is not False - module_path = os.path.join(self.mods_install.dir, "modules", "nf-core", "modules", "trimgalore") + module_path = os.path.join(self.mods_install.dir, "modules", "nf-core", "trimgalore") assert os.path.exists(module_path) @@ -66,6 +66,4 @@ def test_modules_install_different_branch_succeed(self): # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) - assert ( - modules_json.get_module_branch("fastp", GITLAB_URL) == GITLAB_BRANCH_TEST_BRANCH - ) # TODO module, repo_url, install_dir + assert modules_json.get_module_branch("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH diff --git a/tests/modules/list.py b/tests/modules/list.py index 1d18b6da72..1f2b39abf9 100644 --- a/tests/modules/list.py +++ b/tests/modules/list.py @@ -2,7 +2,7 @@ import nf_core.modules -from ..utils import GITLAB_URL +from ..utils import GITLAB_DEFAULT_BRANCH, GITLAB_URL def test_modules_list_remote(self): @@ -17,8 +17,9 @@ def test_modules_list_remote(self): def test_modules_list_remote_gitlab(self): """Test listing the modules in the remote gitlab repo""" - mods_list = nf_core.modules.ModuleList(None, remote=True, remote_url=GITLAB_URL) + mods_list = nf_core.modules.ModuleList(None, remote=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH) listed_mods = mods_list.list_modules() + print(f"listed modules are {listed_mods}") console = Console(record=True) console.print(listed_mods) output = console.export_text() diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 1c6ad0ff31..d13997f32b 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -34,10 +34,13 @@ def test_mod_json_update(self): mod_repo_obj = ModulesRepo() mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", False) mod_json = mod_json_obj.get_modules_json() - assert "MODULE_NAME" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["MODULE_NAME"] - assert "GIT_SHA" == mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["MODULE_NAME"]["git_sha"] - assert NF_CORE_MODULES_DEFAULT_BRANCH == mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["MODULE_NAME"]["branch"] + assert "MODULE_NAME" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"] + assert "GIT_SHA" == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["git_sha"] + assert ( + NF_CORE_MODULES_DEFAULT_BRANCH + == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["branch"] + ) def test_mod_json_create(self): @@ -59,9 +62,9 @@ def test_mod_json_create(self): mods = ["fastqc", "multiqc"] for mod in mods: - assert mod in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][mod] - assert "branch" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"][mod] + assert mod in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"][mod] + assert "branch" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"][mod] def modify_main_nf(path): @@ -79,7 +82,7 @@ def test_mod_json_create_with_patch(self): mod_json_path = Path(self.pipeline_dir, "modules.json") # Modify the module - module_path = Path(self.pipeline_dir, "modules", "nf-core", "modules", "fastqc") + module_path = Path(self.pipeline_dir, "modules", "nf-core", "fastqc") modify_main_nf(module_path / "main.nf") # Try creating a patch file @@ -100,9 +103,9 @@ def test_mod_json_create_with_patch(self): mod_json = mod_json_obj.get_modules_json() # Check that fastqc is in the file - assert "fastqc" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"] - assert "branch" in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"]["fastqc"] + assert "fastqc" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["fastqc"] + assert "branch" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["fastqc"] # Check that fastqc/main.nf maintains the changes with open(module_path / "main.nf", "r") as fh: @@ -162,14 +165,14 @@ def test_mod_json_up_to_date_reinstall_fails(self): mod_json = mod_json_obj.get_modules_json() # Check that the module has been removed from the modules.json - assert "fastqc" not in mod_json["repos"][NF_CORE_MODULES_NAME]["modules"] + assert "fastqc" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] def test_mod_json_repo_present(self): """Tests the repo_present function""" mod_json_obj = ModulesJson(self.pipeline_dir) - assert mod_json_obj.repo_present(NF_CORE_MODULES_NAME) is True + assert mod_json_obj.repo_present(NF_CORE_MODULES_REMOTE) is True assert mod_json_obj.repo_present("INVALID_REPO") is False @@ -177,10 +180,10 @@ def test_mod_json_module_present(self): """Tests the module_present function""" mod_json_obj = ModulesJson(self.pipeline_dir) - assert mod_json_obj.module_present("fastqc", NF_CORE_MODULES_NAME) is True - assert mod_json_obj.module_present("INVALID_MODULE", NF_CORE_MODULES_NAME) is False - assert mod_json_obj.module_present("fastqc", "INVALID_REPO") is False - assert mod_json_obj.module_present("INVALID_MODULE", "INVALID_REPO") is False + assert mod_json_obj.module_present("fastqc", NF_CORE_MODULES_REMOTE, NF_CORE_MODULES_NAME) is True + assert mod_json_obj.module_present("INVALID_MODULE", NF_CORE_MODULES_REMOTE, NF_CORE_MODULES_NAME) is False + assert mod_json_obj.module_present("fastqc", "INVALID_REPO", "INVALID_DIR") is False + assert mod_json_obj.module_present("INVALID_MODULE", "INVALID_REPO", "INVALID_DIR") is False def test_mod_json_get_module_version(self): @@ -188,17 +191,10 @@ def test_mod_json_get_module_version(self): mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() assert ( - mod_json_obj.get_module_version("fastqc", NF_CORE_MODULES_REMOTE) - == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["fastqc"]["git_sha"] + mod_json_obj.get_module_version("fastqc", NF_CORE_MODULES_REMOTE, NF_CORE_MODULES_NAME) + == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["fastqc"]["git_sha"] ) - assert mod_json_obj.get_module_version("INVALID_MODULE", NF_CORE_MODULES_REMOTE) is None - - -def test_mod_json_get_git_url(self): - """Tests the get_git_url function""" - mod_json_obj = ModulesJson(self.pipeline_dir) - assert mod_json_obj.get_git_url(NF_CORE_MODULES_NAME) == NF_CORE_MODULES_REMOTE - assert mod_json_obj.get_git_url("INVALID_REPO") is None + assert mod_json_obj.get_module_version("INVALID_MODULE", NF_CORE_MODULES_REMOTE, NF_CORE_MODULES_NAME) is None def test_mod_json_dump(self): @@ -222,9 +218,10 @@ def test_mod_json_dump(self): def test_mod_json_with_empty_modules_value(self): # Load module.json and remove the modules entry mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_obj.create() # Create modules.json explicitly to get correct module sha mod_json_orig = mod_json_obj.get_modules_json() mod_json = copy.deepcopy(mod_json_orig) - mod_json["repos"]["nf-core/modules"]["modules"] = "" + mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"] = "" # save the altered module.json and load it again to check if it will fix itself mod_json_obj.modules_json = mod_json mod_json_obj.dump() @@ -237,9 +234,10 @@ def test_mod_json_with_empty_modules_value(self): def test_mod_json_with_missing_modules_entry(self): # Load module.json and remove the modules entry mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_obj.create() # Create modules.json explicitly to get correct module sha mod_json_orig = mod_json_obj.get_modules_json() mod_json = copy.deepcopy(mod_json_orig) - mod_json["repos"]["nf-core/modules"].pop("modules") + mod_json["repos"][NF_CORE_MODULES_REMOTE].pop("modules") # save the altered module.json and load it again to check if it will fix itself mod_json_obj.modules_json = mod_json mod_json_obj.dump() diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 0492bd4c07..81c45ee0ae 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -21,8 +21,9 @@ SUCCEED_SHA = "f7d3a3894f67db2e2f3f8c9ba76f8e33356be8e0" FAIL_SHA = "b4596169055700533865cefb7542108418f53100" BISMARK_ALIGN = "bismark/align" -REPO_NAME = "nf-core/modules-test" -PATCH_BRANCH = "patch-tester" +REPO_NAME = "nf-core" +PATCH_BRANCH = "patch-tester-restructure" +REPO_URL = "https://gitlab.com/nf-core/modules-test.git" def setup_patch(pipeline_dir, modify_module): @@ -69,7 +70,7 @@ def test_create_patch_no_change(self): # Check the 'modules.json' contains no patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) is None # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) is None def test_create_patch_change(self): @@ -88,7 +89,7 @@ def test_create_patch_change(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -121,7 +122,7 @@ def test_create_patch_try_apply_successful(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -138,14 +139,14 @@ def test_create_patch_try_apply_successful(self): assert update_obj.try_apply_patch(BISMARK_ALIGN, REPO_NAME, patch_relpath, module_path, module_install_dir) is True # Move the files from the temporary directory - update_obj.move_files_from_tmp_dir(BISMARK_ALIGN, module_path, install_dir, REPO_NAME, SUCCEED_SHA) + update_obj.move_files_from_tmp_dir(BISMARK_ALIGN, install_dir, REPO_NAME, SUCCEED_SHA) # Check that a patch file with the correct name has been created assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -187,7 +188,7 @@ def test_create_patch_try_apply_failed(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -224,7 +225,7 @@ def test_create_patch_update_success(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) @@ -239,16 +240,15 @@ def test_create_patch_update_success(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn - ), modules_json_obj.get_patch_fn( # TODO module_name, repo_url, install_dir - BISMARK_ALIGN, REPO_NAME - ) # TODO module_name, repo_url, install_dir + ), modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) # Check that the correct lines are in the patch file with open(module_path / patch_fn, "r") as fh: patch_lines = fh.readlines() module_relpath = module_path.relative_to(self.pipeline_dir) + print(patch_lines) assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines assert f"+++ {module_relpath / 'main.nf'}\n" in patch_lines assert "- tuple val(meta), path(reads)\n" in patch_lines @@ -282,7 +282,7 @@ def test_create_patch_update_fail(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_NAME) == Path( # TODO module_name, repo_url, install_dir + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) diff --git a/tests/modules/update.py b/tests/modules/update.py index 5ab3917248..10a00c7294 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -226,7 +226,9 @@ def test_update_different_branch_mixed_modules_main(self): # Install MultiQC from gitlab default branch install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH, force=True) - assert install_obj.install("multiqc") # Force as the same module is installed from github nf-core modules repo + assert install_obj.install( + "multiqc" + ) # Force as the same module is installed from github nf-core modules repo by default # Try updating update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) diff --git a/tests/test_modules.py b/tests/test_modules.py index e5cec06c64..214efc6b1c 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -10,15 +10,15 @@ import nf_core.create import nf_core.modules -from .utils import GITLAB_URL, OLD_TRIMGALORE_SHA +from .utils import GITLAB_DEFAULT_BRANCH, GITLAB_URL, OLD_TRIMGALORE_SHA def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" root_dir = os.path.join(tmp_dir, "modules") - os.makedirs(os.path.join(root_dir, "modules")) - os.makedirs(os.path.join(root_dir, "tests", "modules")) + os.makedirs(os.path.join(root_dir, "modules", "nf-core")) + os.makedirs(os.path.join(root_dir, "tests", "modules", "nf-core")) os.makedirs(os.path.join(root_dir, "tests", "config")) with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) @@ -53,7 +53,7 @@ def setUp(self): self.pipeline_dir, prompt=False, force=False, sha=OLD_TRIMGALORE_SHA ) self.mods_install_gitlab = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL + self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH ) # Set up remove objects @@ -133,7 +133,6 @@ def test_modulesrepo_class(self): test_mod_json_create, test_mod_json_create_with_patch, test_mod_json_dump, - test_mod_json_get_git_url, test_mod_json_get_module_version, test_mod_json_module_present, test_mod_json_repo_present, From 95b2d7b3efbf1ea764207d982d0ccedae1f61656 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 3 Oct 2022 11:05:35 +0200 Subject: [PATCH 162/854] Don't skip base.config when skipping nf-core/configs --- nf_core/pipeline-template/nextflow.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index bc222cb6e4..737341c4b0 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -59,10 +59,11 @@ params { max_time = '240.h' } -{% if nf_core_configs %} + // Load base.config by default for all pipelines includeConfig 'conf/base.config' +{% if nf_core_configs -%} // Load nf-core custom profiles from different Institutions try { includeConfig "${params.custom_config_base}/nfcore_custom.config" @@ -77,9 +78,8 @@ try { // } catch (Exception e) { // System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") // } - - {% endif %} + profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { From 11829516ff48c42ca0377751e63458141dde6c54 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 3 Oct 2022 11:06:49 +0200 Subject: [PATCH 163/854] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b238a419f..9721600cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) - Fix template spacing modified by JINJA ([#1830](https://github.com/nf-core/tools/pull/1830)) - Fix MultiQC execution on template [#1855](https://github.com/nf-core/tools/pull/1855) +- Don't skip including `base.config` when skipping nf-core/configs ### Linting From f7bdf4171fac4b736946a11d83f93048be977004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 3 Oct 2022 12:22:46 +0000 Subject: [PATCH 164/854] fix sha tests --- nf_core/modules/modules_repo.py | 6 +- nf_core/modules/patch.py | 2 +- nf_core/modules/update.py | 11 +-- nf_core/pipeline-template/modules.json | 6 +- .../custom/dumpsoftwareversions/main.nf | 8 +- .../modules/nf-core/fastqc/main.nf | 12 +++ .../modules/nf-core/multiqc/main.nf | 2 +- tests/modules/modules_json.py | 2 +- tests/modules/patch.py | 16 ++-- tests/modules/update.py | 90 ++++++++----------- tests/test_modules.py | 17 +++- tests/utils.py | 7 +- 12 files changed, 99 insertions(+), 80 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 5551bdaa9d..9458fce723 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -310,8 +310,8 @@ def install_module(self, module_name, install_dir, commit): return False # Check if the module exists in the branch - if not self.module_exists(module_name): - log.error(f"The requested module does not exists in the '{self.branch}' of {self.remote_url}'") + if not self.module_exists(module_name, checkout=False): + log.error(f"The requested module does not exists in the branch '{self.branch}' of {self.remote_url}'") return False # Copy the files from the repo to the install folder @@ -364,7 +364,7 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 ( dict ): Iterator of commit SHAs and associated (truncated) message """ self.checkout_branch() - module_path = os.path.join("modules", module_name) + module_path = os.path.join("modules", self.repo_path, module_name) commits = self.repo.iter_commits(max_count=depth, paths=module_path) commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) return commits diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 670d7ec108..890b0d4c49 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -85,7 +85,7 @@ def patch(self, module=None): module_install_dir = Path(install_dir, module) if not self.install_module_files(module, module_version, self.modules_repo, install_dir): raise UserWarning( - f"Failed to install files of module '{module_fullname}' from remote ({self.modules_repo.remote_url})." + f"Failed to install files of module '{module}' from remote ({self.modules_repo.remote_url})." ) # Write the patch to a temporary location (otherwise it is printed to the screen later) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index e6b58e717f..e1dd2cc022 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -428,13 +428,13 @@ def get_all_modules_info(self, branch=None): ) ) except KeyError: - modules_info[repo_name][module_dir].append( + modules_info[repo_name][module_dir] = [ ( module, custom_sha, self.modules_json.get_module_branch(module, repo_name, module_dir), ) - ) + ] if self.sha is not None: overridden_modules.append(module) elif dir_config[module] is False: @@ -574,15 +574,16 @@ def move_files_from_tmp_dir(self, module, install_folder, repo_path, new_version """ temp_module_dir = os.path.join(install_folder, module) files = os.listdir(temp_module_dir) + pipeline_path = os.path.join(self.dir, "modules", repo_path, module) log.debug(f"Removing old version of module '{module}'") - self.clear_module_dir(module, repo_path) + self.clear_module_dir(module, pipeline_path) - os.makedirs(repo_path) + os.makedirs(pipeline_path) for file in files: path = os.path.join(temp_module_dir, file) if os.path.exists(path): - shutil.move(path, os.path.join(repo_path, file)) + shutil.move(path, os.path.join(pipeline_path, file)) log.info(f"Updating '{repo_path}/{module}'") log.debug(f"Updating module '{module}' to {new_version} from {repo_path}") diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 21644f7c60..06008b510c 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,15 +7,15 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "restructure", - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" }, "fastqc": { "branch": "restructure", - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" }, "multiqc": { "branch": "restructure", - "git_sha": "4b1d4bf401d4cf65ebc4f604bc8c8e7551109db3" + "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" } } } diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index 327d510056..34b50b9f84 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -1,11 +1,11 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_low' + label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? "bioconda::multiqc=1.11" : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13a' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.11--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13a--pyhdfd78af_1' : + 'quay.io/biocontainers/multiqc:1.13a--pyhdfd78af_1' }" input: path versions diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index ed6b8c50b1..05730368b2 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -44,4 +44,16 @@ process FASTQC { END_VERSIONS """ } + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.html + touch ${prefix}.zip + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ } diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index 698461d7b4..a8159a57bf 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -1,5 +1,5 @@ process MULTIQC { - label 'process_medium' + label 'process_single' conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index d13997f32b..60fb3006c1 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -86,7 +86,7 @@ def test_mod_json_create_with_patch(self): modify_main_nf(module_path / "main.nf") # Try creating a patch file - patch_obj = ModulePatch(self.pipeline_dir) + patch_obj = ModulePatch(self.pipeline_dir, NF_CORE_MODULES_REMOTE, NF_CORE_MODULES_DEFAULT_BRANCH) patch_obj.patch("fastqc") # Remove the existing modules.json file diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 81c45ee0ae..44403f4624 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -17,9 +17,9 @@ testing if the update commands works correctly with patch files """ -ORG_SHA = "22c7c12dc21e2f633c00862c1291ceda0a3b7066" -SUCCEED_SHA = "f7d3a3894f67db2e2f3f8c9ba76f8e33356be8e0" -FAIL_SHA = "b4596169055700533865cefb7542108418f53100" +ORG_SHA = "775fcd090fb776a0be695044f8ab1af8896c8452" +SUCCEED_SHA = "f1566140c752e9c68fffc189fbe8cb9ee942b3ca" +FAIL_SHA = "1fc8b0f953d915d66ee40d28bc337ff0998d05bd" BISMARK_ALIGN = "bismark/align" REPO_NAME = "nf-core" PATCH_BRANCH = "patch-tester-restructure" @@ -28,7 +28,7 @@ def setup_patch(pipeline_dir, modify_module): install_obj = nf_core.modules.ModuleInstall( - pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA + pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA ) # Install the module @@ -48,8 +48,12 @@ def modify_main_nf(path): # - tuple val(meta), path(reads) # - path index # + tuple val(meta), path(reads), path(index) - lines[10] = " tuple val(meta), path(reads), path(index)\n" - lines.pop(11) + for line_index in range(len(lines)): + if lines[line_index] == " tuple val(meta), path(reads)\n": + lines[line_index] = " tuple val(meta), path(reads), path(index)\n" + elif lines[line_index] == " path index\n": + to_pop = line_index + lines.pop(to_pop) with open(path, "w") as fh: fh.writelines(lines) diff --git a/tests/modules/update.py b/tests/modules/update.py index 10a00c7294..c159a0e362 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -19,6 +19,7 @@ GITLAB_REPO, GITLAB_URL, OLD_TRIMGALORE_SHA, + OLD_TRIMGALORE_BRANCH, ) @@ -39,13 +40,13 @@ def test_install_and_update(self): def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" - self.mods_install_old.install("trimgalore") - update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) + assert self.mods_install_old.install("trimgalore") + update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) # Copy the module files and check that they are affected by the update tmpdir = tempfile.mkdtemp() trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") shutil.copytree(trimgalore_path, trimgalore_tmpdir) assert update_obj.update("trimgalore") is True @@ -56,7 +57,7 @@ def test_install_at_hash_and_update(self): mod_json = mod_json_obj.get_modules_json() # Get the up-to-date git_sha for the module from the ModulesRepo object correct_git_sha = update_obj.modules_repo.get_latest_module_version("trimgalore") - current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"][ + current_git_sha = mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"][ "git_sha" ] assert correct_git_sha == current_git_sha @@ -66,12 +67,12 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a module in the pipeline and updates it""" self.mods_install_old.install("trimgalore") patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") - update_obj = ModuleUpdate(self.pipeline_dir, save_diff_fn=patch_path) + update_obj = ModuleUpdate(self.pipeline_dir, save_diff_fn=patch_path, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) # Copy the module files and check that they are affected by the update tmpdir = tempfile.mkdtemp() trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") shutil.copytree(trimgalore_path, trimgalore_tmpdir) assert update_obj.update("trimgalore") is True @@ -99,25 +100,25 @@ def test_update_all(self): def test_update_with_config_fixed_version(self): """Try updating when there are entries in the .nf-core.yml""" # Install trimgalore at the latest version - self.mods_install.install("trimgalore") + assert self.mods_install_trimgalore.install("trimgalore") # Fix the trimgalore version in the .nf-core.yml to an old version - update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"trimgalore": OLD_TRIMGALORE_SHA}}} + update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": OLD_TRIMGALORE_SHA}}} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"] + assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] assert ( - mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"]["git_sha"] + mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA ) @@ -128,46 +129,46 @@ def test_update_with_config_dont_update(self): self.mods_install_old.install("trimgalore") # Set the trimgalore field to no update in the .nf-core.yml - update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"trimgalore": False}}} + update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": False}}} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "trimgalore" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"] + assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] assert ( - mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"]["git_sha"] + mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA ) def test_update_with_config_fix_all(self): """Fix the version of all nf-core modules""" - self.mods_install.install("trimgalore") + self.mods_install_trimgalore.install("trimgalore") # Fix the version of all nf-core modules in the .nf-core.yml to an old version - update_config = {NF_CORE_MODULES_REMOTE: OLD_TRIMGALORE_SHA} + update_config = {GITLAB_URL: OLD_TRIMGALORE_SHA} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"] + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] assert ( - mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["trimgalore"]["git_sha"] + mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA ) @@ -178,33 +179,33 @@ def test_update_with_config_no_updates(self): old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Fix the version of all nf-core modules in the .nf-core.yml to an old version - update_config = {NF_CORE_MODULES_REMOTE: False} + update_config = {GITLAB_URL: False} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded and none of the modules has changed mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - for module in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: - assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][module] + for module in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]: + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module] assert ( - mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][module]["git_sha"] - == old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][module]["git_sha"] + mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module]["git_sha"] + == old_mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module]["git_sha"] ) def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" - install_obj = ModuleInstall( - self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + install_obj = nf_core.modules.ModuleInstall( + self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA ) - installed = install_obj.install("fastp") - assert installed + assert install_obj.install("fastp") + update_obj = ModuleUpdate( self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False ) @@ -219,16 +220,10 @@ def test_update_different_branch_single_module(self): def test_update_different_branch_mixed_modules_main(self): """Try updating all modules where MultiQC is installed from main branch""" # Install fastp - install_obj = ModuleInstall( - self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA - ) - assert install_obj.install("fastp") + assert self.mods_install_gitlab_old.install("fastp") # Install MultiQC from gitlab default branch - install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH, force=True) - assert install_obj.install( - "multiqc" - ) # Force as the same module is installed from github nf-core modules repo by default + assert self.mods_install_gitlab.install("multiqc") # Try updating update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) @@ -245,18 +240,11 @@ def test_update_different_branch_mixed_modules_main(self): def test_update_different_branch_mix_modules_branch_test(self): """Try updating all modules where MultiQC is installed from branch-test branch""" # Install multiqc from the branch-test branch - install_obj = ModuleInstall( - self.pipeline_dir, - remote_url=GITLAB_URL, - branch=GITLAB_BRANCH_TEST_BRANCH, - sha=GITLAB_BRANCH_TEST_OLD_SHA, - force=True, - ) - install_obj.install("multiqc") # Force as the same module is installed from github nf-core modules repo - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) - update_obj.update() - + assert self.mods_install_gitlab_old.install("multiqc") # Force as the same module is installed from github nf-core modules repo modules_json = ModulesJson(self.pipeline_dir) + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_NEW_SHA) + assert update_obj.update() + assert modules_json.get_module_branch("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH assert modules_json.get_module_version("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA diff --git a/tests/test_modules.py b/tests/test_modules.py index 214efc6b1c..f2f8a42922 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -10,7 +10,14 @@ import nf_core.create import nf_core.modules -from .utils import GITLAB_DEFAULT_BRANCH, GITLAB_URL, OLD_TRIMGALORE_SHA +from .utils import ( + GITLAB_DEFAULT_BRANCH, + GITLAB_URL, + OLD_TRIMGALORE_SHA, + OLD_TRIMGALORE_BRANCH, + GITLAB_BRANCH_TEST_BRANCH, + GITLAB_BRANCH_TEST_OLD_SHA, +) def create_modules_repo_dummy(tmp_dir): @@ -50,11 +57,17 @@ def setUp(self): self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) self.mods_install_old = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=False, sha=OLD_TRIMGALORE_SHA + self.pipeline_dir, prompt=False, force=False, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + ) + self.mods_install_trimgalore = nf_core.modules.ModuleInstall( + self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH ) self.mods_install_gitlab = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH ) + self.mods_install_gitlab_old = nf_core.modules.ModuleInstall( + self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + ) # Set up remove objects self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) diff --git a/tests/utils.py b/tests/utils.py index 850b151ed5..fb592a6209 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,14 +10,15 @@ from contextlib import contextmanager from pathlib import Path -OLD_TRIMGALORE_SHA = "20d8250d9f39ddb05dfb437603aaf99b5c0b2b41" +OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" +OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" GITLAB_REPO = "nf-core" GITLAB_DEFAULT_BRANCH = "main-restructure" # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester-restructure" -GITLAB_BRANCH_TEST_OLD_SHA = "eb4bc244de7eaef8e8ff0d451e4ca2e4b2c29821" -GITLAB_BRANCH_TEST_NEW_SHA = "a4ac8e1abb05dcd583da18208f2ca951de2a205e" +GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" +GITLAB_BRANCH_TEST_NEW_SHA = "2f5f180f6e705bb81d6e7742dc2f24bf4a0c721e" def with_temporary_folder(func): From 58f14914bb44762f96c4bcd0a4fb525c09d7c6c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 3 Oct 2022 12:35:31 +0000 Subject: [PATCH 165/854] fix gitlab workflow --- .github/workflows/create-lint-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 277baf1425..102aaa4ea1 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -102,7 +102,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git install fastqc --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main-restructure install fastqc --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ From 94231c49ab4e4a37f37199cdcb54bd35f258528c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 3 Oct 2022 14:44:13 +0200 Subject: [PATCH 166/854] fix linting --- tests/modules/update.py | 75 ++++++++++++++++++++++++++++------------- tests/test_modules.py | 20 ++++++++--- 2 files changed, 66 insertions(+), 29 deletions(-) diff --git a/tests/modules/update.py b/tests/modules/update.py index c159a0e362..bb1da50f52 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -18,8 +18,8 @@ GITLAB_DEFAULT_BRANCH, GITLAB_REPO, GITLAB_URL, - OLD_TRIMGALORE_SHA, OLD_TRIMGALORE_BRANCH, + OLD_TRIMGALORE_SHA, ) @@ -57,9 +57,7 @@ def test_install_at_hash_and_update(self): mod_json = mod_json_obj.get_modules_json() # Get the up-to-date git_sha for the module from the ModulesRepo object correct_git_sha = update_obj.modules_repo.get_latest_module_version("trimgalore") - current_git_sha = mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"][ - "git_sha" - ] + current_git_sha = mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] assert correct_git_sha == current_git_sha @@ -67,7 +65,13 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a module in the pipeline and updates it""" self.mods_install_old.install("trimgalore") patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") - update_obj = ModuleUpdate(self.pipeline_dir, save_diff_fn=patch_path, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) + update_obj = ModuleUpdate( + self.pipeline_dir, + save_diff_fn=patch_path, + sha=OLD_TRIMGALORE_SHA, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + ) # Copy the module files and check that they are affected by the update tmpdir = tempfile.mkdtemp() @@ -110,17 +114,16 @@ def test_update_with_config_fixed_version(self): yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) + update_obj = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + ) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] - assert ( - mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] - == OLD_TRIMGALORE_SHA - ) + assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA def test_update_with_config_dont_update(self): @@ -136,17 +139,21 @@ def test_update_with_config_dont_update(self): yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) + update_obj = ModuleUpdate( + self.pipeline_dir, + update_all=True, + show_diff=False, + sha=OLD_TRIMGALORE_SHA, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + ) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] - assert ( - mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] - == OLD_TRIMGALORE_SHA - ) + assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA def test_update_with_config_fix_all(self): @@ -161,16 +168,15 @@ def test_update_with_config_fix_all(self): yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) + update_obj = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + ) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] - assert ( - mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] - == OLD_TRIMGALORE_SHA - ) + assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA def test_update_with_config_no_updates(self): @@ -186,7 +192,14 @@ def test_update_with_config_no_updates(self): yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) + update_obj = ModuleUpdate( + self.pipeline_dir, + update_all=True, + show_diff=False, + sha=OLD_TRIMGALORE_SHA, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + ) assert update_obj.update() is True # Check that the git sha for trimgalore is correctly downgraded and none of the modules has changed @@ -202,7 +215,12 @@ def test_update_with_config_no_updates(self): def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" install_obj = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + self.pipeline_dir, + prompt=False, + force=False, + remote_url=GITLAB_URL, + branch=GITLAB_BRANCH_TEST_BRANCH, + sha=GITLAB_BRANCH_TEST_OLD_SHA, ) assert install_obj.install("fastp") @@ -240,9 +258,18 @@ def test_update_different_branch_mixed_modules_main(self): def test_update_different_branch_mix_modules_branch_test(self): """Try updating all modules where MultiQC is installed from branch-test branch""" # Install multiqc from the branch-test branch - assert self.mods_install_gitlab_old.install("multiqc") # Force as the same module is installed from github nf-core modules repo + assert self.mods_install_gitlab_old.install( + "multiqc" + ) # Force as the same module is installed from github nf-core modules repo modules_json = ModulesJson(self.pipeline_dir) - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_NEW_SHA) + update_obj = ModuleUpdate( + self.pipeline_dir, + update_all=True, + show_diff=False, + remote_url=GITLAB_URL, + branch=GITLAB_BRANCH_TEST_BRANCH, + sha=GITLAB_BRANCH_TEST_NEW_SHA, + ) assert update_obj.update() assert modules_json.get_module_branch("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH diff --git a/tests/test_modules.py b/tests/test_modules.py index f2f8a42922..a16c9e6328 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -11,12 +11,12 @@ import nf_core.modules from .utils import ( + GITLAB_BRANCH_TEST_BRANCH, + GITLAB_BRANCH_TEST_OLD_SHA, GITLAB_DEFAULT_BRANCH, GITLAB_URL, - OLD_TRIMGALORE_SHA, OLD_TRIMGALORE_BRANCH, - GITLAB_BRANCH_TEST_BRANCH, - GITLAB_BRANCH_TEST_OLD_SHA, + OLD_TRIMGALORE_SHA, ) @@ -57,7 +57,12 @@ def setUp(self): self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) self.mods_install_old = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=False, sha=OLD_TRIMGALORE_SHA, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + self.pipeline_dir, + prompt=False, + force=False, + sha=OLD_TRIMGALORE_SHA, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, ) self.mods_install_trimgalore = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH @@ -66,7 +71,12 @@ def setUp(self): self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH ) self.mods_install_gitlab_old = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA + self.pipeline_dir, + prompt=False, + force=True, + remote_url=GITLAB_URL, + branch=GITLAB_BRANCH_TEST_BRANCH, + sha=GITLAB_BRANCH_TEST_OLD_SHA, ) # Set up remove objects From 3a02093c64f113572662d8bf6f58636d6c5fdd67 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 3 Oct 2022 15:10:35 +0200 Subject: [PATCH 167/854] gitlab force --- .github/workflows/create-lint-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 102aaa4ea1..c0a721b868 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -102,7 +102,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main-restructure install fastqc --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main-restructure install fastqc --force --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ From e96386e0cbcda9e36a600eb753dd5d167cc4f84f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 3 Oct 2022 15:13:02 +0000 Subject: [PATCH 168/854] fix yaml sha --- tests/modules/lint.py | 19 +++++++++++++++++-- tests/modules/patch.py | 13 +++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index df3a4cbe24..d7a2821d3b 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -1,11 +1,26 @@ import os +from pathlib import Path import pytest import nf_core.modules from ..utils import GITLAB_URL, set_wd -from .patch import BISMARK_ALIGN, PATCH_BRANCH, setup_patch +from .patch import BISMARK_ALIGN, PATCH_BRANCH, CORRECT_SHA, REPO_NAME, modify_main_nf + + +def setup_patch(pipeline_dir, modify_module): + install_obj = nf_core.modules.ModuleInstall( + pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=CORRECT_SHA + ) + + # Install the module + install_obj.install(BISMARK_ALIGN) + + if modify_module: + # Modify the module + module_path = Path(pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) + modify_main_nf(module_path / "main.nf") def test_modules_lint_trimgalore(self): @@ -66,7 +81,7 @@ def test_modules_lint_patched_modules(self): # change temporarily working directory to the pipeline directory # to avoid error from try_apply_patch() during linting with set_wd(self.pipeline_dir): - module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL, branch=PATCH_BRANCH) module_lint.lint(print_results=False, all_modules=True) assert len(module_lint.failed) == 0 diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 44403f4624..fe736c447a 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -18,6 +18,7 @@ """ ORG_SHA = "775fcd090fb776a0be695044f8ab1af8896c8452" +CORRECT_SHA = "335cd32405568ca3b6d4c05ab1e8a98c21e18a4d" SUCCEED_SHA = "f1566140c752e9c68fffc189fbe8cb9ee942b3ca" FAIL_SHA = "1fc8b0f953d915d66ee40d28bc337ff0998d05bd" BISMARK_ALIGN = "bismark/align" @@ -229,24 +230,28 @@ def test_create_patch_update_success(self): # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, GITLAB_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) + with open(os.path.join(module_path, "main.nf"), "r") as fh: + print(fh.readlines()) # Update the module update_obj = nf_core.modules.ModuleUpdate( self.pipeline_dir, sha=SUCCEED_SHA, show_diff=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH ) - update_obj.update(BISMARK_ALIGN) + assert update_obj.update(BISMARK_ALIGN) + with open(os.path.join(module_path, "main.nf"), "r") as fh: + print(fh.readlines()) # Check that a patch file with the correct name has been created assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} # Check the 'modules.json' contains a patch file for the module modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) - assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, GITLAB_URL, REPO_NAME) == Path( "modules", REPO_NAME, BISMARK_ALIGN, patch_fn - ), modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) + ), modules_json_obj.get_patch_fn(BISMARK_ALIGN, GITLAB_URL, REPO_NAME) # Check that the correct lines are in the patch file with open(module_path / patch_fn, "r") as fh: From dd7fb4f0fcb6304a2dcd1527abed19aad9250cd0 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 3 Oct 2022 17:18:45 +0200 Subject: [PATCH 169/854] fix isort --- tests/modules/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index d7a2821d3b..d063188073 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -6,7 +6,7 @@ import nf_core.modules from ..utils import GITLAB_URL, set_wd -from .patch import BISMARK_ALIGN, PATCH_BRANCH, CORRECT_SHA, REPO_NAME, modify_main_nf +from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf def setup_patch(pipeline_dir, modify_module): From 79a48fd69c40ab150902da1d0abed5a70da36055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 4 Oct 2022 09:12:17 +0200 Subject: [PATCH 170/854] remove branch and url from specific modules command --- nf_core/__main__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0feac32391..9df0b2ff97 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -398,9 +398,7 @@ def list(ctx): @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -@click.option("-u", "--url", help="Remote repository url to list from") -@click.option("-b", "--branch", help="Remote repository branch to list") -def remote(ctx, keywords, json, url, branch): +def remote(ctx, keywords, json): """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -408,8 +406,8 @@ def remote(ctx, keywords, json, url, branch): module_list = nf_core.modules.ModuleList( None, True, - ctx.obj["modules_repo_branch"] if url is None else url, - ctx.obj["modules_repo_branch"] if branch is None else branch, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) stdout.print(module_list.list_modules(keywords, json)) From 9ddc614c7b42d7b501181a1593140109b14504ec Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 10:54:18 +0200 Subject: [PATCH 171/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9721600cc1..c5230203d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Use contextlib to temporarily change working directories ([#1819](https://github.com/nf-core/tools/pull/1819)) - More helpful error messages if `nf-core download` can't parse a singularity image download - Modules: If something is wrong with the local repo cache, offer to delete it and try again ([#1850](https://github.com/nf-core/tools/issues/1850)) +- Restructure code to work with the directory restructuring in [modules](https://github.com/nf-core/modules/pull/2141) ([#1859](https://github.com/nf-core/tools/pull/1859)) ### Modules From 683568aaa73450539cc0e8d26ce8b284bed4fdba Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 10:55:19 +0200 Subject: [PATCH 172/854] change nf-core/modules branch to master --- nf_core/modules/modules_repo.py | 4 ++-- nf_core/pipeline-template/modules.json | 6 +++--- tests/test_modules.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 1f607b7475..b21dd0b9d0 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -19,7 +19,7 @@ # Constants for the nf-core/modules repo used throughout the module files NF_CORE_MODULES_NAME = "nf-core" NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" -NF_CORE_MODULES_DEFAULT_BRANCH = "restructure" +NF_CORE_MODULES_DEFAULT_BRANCH = "master" class RemoteProgressbar(git.RemoteProgress): @@ -223,7 +223,7 @@ def setup_branch(self, branch): if branch is None: # Don't bother fetching default branch if we're using nf-core if self.remote_url == NF_CORE_MODULES_REMOTE: - self.branch = "restructure" + self.branch = "master" else: self.branch = self.get_default_branch() else: diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 06008b510c..53d85d4f39 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -6,15 +6,15 @@ "modules": { "nf-core": { "custom/dumpsoftwareversions": { - "branch": "restructure", + "branch": "master", "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" }, "fastqc": { - "branch": "restructure", + "branch": "master", "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" }, "multiqc": { - "branch": "restructure", + "branch": "master", "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" } } diff --git a/tests/test_modules.py b/tests/test_modules.py index a16c9e6328..8fb9bab31b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -96,7 +96,7 @@ def test_modulesrepo_class(self): """Initialise a modules repo object""" modrepo = nf_core.modules.ModulesRepo() assert modrepo.repo_path == "nf-core" - assert modrepo.branch == "restructure" + assert modrepo.branch == "master" ############################################ # Test of the individual modules commands. # From cb0ab31527d53939cb54bad9ff213d76930caf23 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 11:31:06 +0200 Subject: [PATCH 173/854] change sha for master branch --- nf_core/pipeline-template/modules.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 53d85d4f39..4037d5905f 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,15 +7,15 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" }, "fastqc": { "branch": "master", - "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" }, "multiqc": { "branch": "master", - "git_sha": "86e32684ce5b13ca8173a78cd2250131ca9b1c0f" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" } } } From ee8459a6649fa24c041ed2371aacd426f7f4f56c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 11:37:04 +0200 Subject: [PATCH 174/854] update custom/dumpsoftwareversions to last commit --- .../modules/nf-core/custom/dumpsoftwareversions/main.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index 34b50b9f84..cebb6e0589 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? 'bioconda::multiqc=1.13a' : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13a--pyhdfd78af_1' : - 'quay.io/biocontainers/multiqc:1.13a--pyhdfd78af_1' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" input: path versions From 39c17d478ffbe7e35d318cfb655b10c4c52e03f2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 12:07:47 +0200 Subject: [PATCH 175/854] bump to v2.6 --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5230203d5..b7402436e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v2.6dev +## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] ### Template diff --git a/setup.py b/setup.py index 0759633a0b..b0ffe22f6d 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.6dev" +version = "2.6" with open("README.md") as f: readme = f.read() From 7cb93589998f5074ba2e3cb994ad021c7b2b0577 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 4 Oct 2022 10:54:54 +0000 Subject: [PATCH 176/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 97 ++++--- docs/images/nf-core-create.svg | 137 +++++---- docs/images/nf-core-download-tree.svg | 146 +++++----- docs/images/nf-core-download.svg | 95 +++---- docs/images/nf-core-launch-rnaseq.svg | 91 +++--- docs/images/nf-core-licences.svg | 139 +++++----- docs/images/nf-core-lint.svg | 278 ++++++++----------- docs/images/nf-core-list-rna.svg | 125 ++++----- docs/images/nf-core-list-stars.svg | 108 +++---- docs/images/nf-core-list.svg | 112 ++++---- docs/images/nf-core-modules-bump-version.svg | 113 ++++---- docs/images/nf-core-modules-create-test.svg | 114 ++++---- docs/images/nf-core-modules-create.svg | 141 +++++----- docs/images/nf-core-modules-info.svg | 211 +++++++------- docs/images/nf-core-modules-install.svg | 95 +++---- docs/images/nf-core-modules-lint.svg | 206 +++++++------- docs/images/nf-core-modules-list-local.svg | 136 ++++----- docs/images/nf-core-modules-list-remote.svg | 137 ++++----- docs/images/nf-core-modules-mulled.svg | 87 +++--- docs/images/nf-core-modules-patch.svg | 157 +++-------- docs/images/nf-core-modules-remove.svg | 79 +++--- docs/images/nf-core-modules-test.svg | 83 +++--- docs/images/nf-core-modules-update.svg | 95 +++---- docs/images/nf-core-schema-build.svg | 91 +++--- docs/images/nf-core-schema-lint.svg | 85 +++--- docs/images/nf-core-schema-validate.svg | 89 +++--- docs/images/nf-core-sync.svg | 91 +++--- 27 files changed, 1588 insertions(+), 1750 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index 3cea80c40f..aed99cd069 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,83 +19,80 @@ font-weight: 700; } - .terminal-218664248-matrix { + .terminal-1201319354-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-218664248-title { + .terminal-1201319354-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-218664248-r1 { fill: #c5c8c6 } -.terminal-218664248-r2 { fill: #98a84b } -.terminal-218664248-r3 { fill: #9a9b99 } -.terminal-218664248-r4 { fill: #608ab1 } -.terminal-218664248-r5 { fill: #d0b344 } -.terminal-218664248-r6 { fill: #868887 } -.terminal-218664248-r7 { fill: #cc555a } + .terminal-1201319354-r1 { fill: #c5c8c6 } +.terminal-1201319354-r2 { fill: #98a84b } +.terminal-1201319354-r3 { fill: #9a9b99 } +.terminal-1201319354-r4 { fill: #608ab1 } +.terminal-1201319354-r5 { fill: #d0b344 } +.terminal-1201319354-r6 { fill: #868887 } +.terminal-1201319354-r7 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -104,27 +101,27 @@ - - - - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 -INFO     Updated version in 'nextflow.config'bump_version.py:164 - - version         = '1.0dev' - + version = '1.1' - - + + + + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1'bump_version.py:35 +INFO     Updated version in 'nextflow.config'bump_version.py:164 + - version         = '1.0dev' + + version = '1.1' + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 2fdc88fd52..e9731ad1a3 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,113 +19,110 @@ font-weight: 700; } - .terminal-1555017928-matrix { + .terminal-70990144-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1555017928-title { + .terminal-70990144-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1555017928-r1 { fill: #c5c8c6 } -.terminal-1555017928-r2 { fill: #98a84b } -.terminal-1555017928-r3 { fill: #9a9b99 } -.terminal-1555017928-r4 { fill: #608ab1 } -.terminal-1555017928-r5 { fill: #d0b344 } -.terminal-1555017928-r6 { fill: #868887 } -.terminal-1555017928-r7 { fill: #98729f } -.terminal-1555017928-r8 { fill: #ff2c7a } -.terminal-1555017928-r9 { fill: #98a84b;font-weight: bold } -.terminal-1555017928-r10 { fill: #1984e9;text-decoration: underline; } + .terminal-70990144-r1 { fill: #c5c8c6 } +.terminal-70990144-r2 { fill: #98a84b } +.terminal-70990144-r3 { fill: #9a9b99 } +.terminal-70990144-r4 { fill: #608ab1 } +.terminal-70990144-r5 { fill: #d0b344 } +.terminal-70990144-r6 { fill: #868887 } +.terminal-70990144-r7 { fill: #98729f } +.terminal-70990144-r8 { fill: #ff2c7a } +.terminal-70990144-r9 { fill: #98a84b;font-weight: bold } +.terminal-70990144-r10 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -134,36 +131,36 @@ - - + + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.1 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 -INFO     Initialising pipeline git repository                                          create.py:538 -INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 -         syncing.                                                                       -INFO    !!!!!! IMPORTANT !!!!!!create.py:227 - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 +INFO     Initialising pipeline git repository                                          create.py:538 +INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 +         syncing.                                                                       +INFO    !!!!!! IMPORTANT !!!!!!create.py:227 + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download-tree.svg b/docs/images/nf-core-download-tree.svg index 24a0f671fe..a06c743fdf 100644 --- a/docs/images/nf-core-download-tree.svg +++ b/docs/images/nf-core-download-tree.svg @@ -19,123 +19,123 @@ font-weight: 700; } - .terminal-187724284-matrix { + .terminal-3113317903-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-187724284-title { + .terminal-3113317903-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-187724284-r1 { fill: #c5c8c6 } + .terminal-3113317903-r1 { fill: #c5c8c6 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -147,43 +147,43 @@ - + - - $ tree -L 2 nf-core-rnaseq/ -nf-core-rnaseq/ -├── configs -│   ├── CITATION.cff -│   ├── LICENSE -│   ├── README.md -│   ├── bin -│   ├── conf -│   ├── configtest.nf -│   ├── docs -│   ├── nextflow.config -│   ├── nfcore_custom.config -│   └── pipeline -└── workflow -    ├── CHANGELOG.md -    ├── CITATIONS.md -    ├── CODE_OF_CONDUCT.md -    ├── LICENSE -    ├── README.md -    ├── assets -    ├── bin -    ├── conf -    ├── docs -    ├── lib -    ├── main.nf -    ├── modules -    ├── modules.json -    ├── nextflow.config -    ├── nextflow_schema.json -    ├── subworkflows -    ├── tower.yml -    └── workflows - -14 directories, 16 files + + $ tree -L 2 nf-core-rnaseq/ +nf-core-rnaseq/ +├── configs +│   ├── CITATION.cff +│   ├── LICENSE +│   ├── README.md +│   ├── bin +│   ├── conf +│   ├── configtest.nf +│   ├── docs +│   ├── nextflow.config +│   ├── nfcore_custom.config +│   └── pipeline +└── workflow +    ├── CHANGELOG.md +    ├── CITATIONS.md +    ├── CODE_OF_CONDUCT.md +    ├── LICENSE +    ├── README.md +    ├── assets +    ├── bin +    ├── conf +    ├── docs +    ├── lib +    ├── main.nf +    ├── modules +    ├── modules.json +    ├── nextflow.config +    ├── nextflow_schema.json +    ├── subworkflows +    ├── tower.yml +    └── workflows + +14 directories, 16 files diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index f26d134197..51be23194a 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,82 +19,79 @@ font-weight: 700; } - .terminal-2067913912-matrix { + .terminal-1992655221-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2067913912-title { + .terminal-1992655221-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2067913912-r1 { fill: #c5c8c6 } -.terminal-2067913912-r2 { fill: #98a84b } -.terminal-2067913912-r3 { fill: #9a9b99 } -.terminal-2067913912-r4 { fill: #608ab1 } -.terminal-2067913912-r5 { fill: #d0b344 } -.terminal-2067913912-r6 { fill: #868887 } + .terminal-1992655221-r1 { fill: #c5c8c6 } +.terminal-1992655221-r2 { fill: #98a84b } +.terminal-1992655221-r3 { fill: #9a9b99 } +.terminal-1992655221-r4 { fill: #608ab1 } +.terminal-1992655221-r5 { fill: #d0b344 } +.terminal-1992655221-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -103,27 +100,27 @@ - - - - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq'download.py:158 -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                                      download.py:161 -INFO     Downloading centralised configs from GitHub                                 download.py:165 + + + + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq'download.py:159 +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                                      download.py:162 +INFO     Downloading centralised configs from GitHub                                 download.py:166 diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index 56ae23fba1..a52e1d7891 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,78 +19,75 @@ font-weight: 700; } - .terminal-2752685693-matrix { + .terminal-100502750-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2752685693-title { + .terminal-100502750-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2752685693-r1 { fill: #c5c8c6 } -.terminal-2752685693-r2 { fill: #98a84b } -.terminal-2752685693-r3 { fill: #9a9b99 } -.terminal-2752685693-r4 { fill: #608ab1 } -.terminal-2752685693-r5 { fill: #d0b344 } -.terminal-2752685693-r6 { fill: #868887 } -.terminal-2752685693-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2752685693-r8 { fill: #68a0b3;font-weight: bold } + .terminal-100502750-r1 { fill: #c5c8c6 } +.terminal-100502750-r2 { fill: #98a84b } +.terminal-100502750-r3 { fill: #9a9b99 } +.terminal-100502750-r4 { fill: #608ab1 } +.terminal-100502750-r5 { fill: #d0b344 } +.terminal-100502750-r6 { fill: #868887 } +.terminal-100502750-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-100502750-r8 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -99,25 +96,25 @@ - - - - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 -         Nextflow config files or profiles                                              - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 + + + + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 +         Nextflow config files or profiles                                              + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 931dd9862e..4173e2faa0 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,114 +19,111 @@ font-weight: 700; } - .terminal-2684361463-matrix { + .terminal-1928727807-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2684361463-title { + .terminal-1928727807-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2684361463-r1 { fill: #c5c8c6 } -.terminal-2684361463-r2 { fill: #98a84b } -.terminal-2684361463-r3 { fill: #9a9b99 } -.terminal-2684361463-r4 { fill: #608ab1 } -.terminal-2684361463-r5 { fill: #d0b344 } -.terminal-2684361463-r6 { fill: #68a0b3;font-weight: bold } -.terminal-2684361463-r7 { fill: #868887 } -.terminal-2684361463-r8 { fill: #c5c8c6;font-weight: bold } + .terminal-1928727807-r1 { fill: #c5c8c6 } +.terminal-1928727807-r2 { fill: #98a84b } +.terminal-1928727807-r3 { fill: #9a9b99 } +.terminal-1928727807-r4 { fill: #608ab1 } +.terminal-1928727807-r5 { fill: #d0b344 } +.terminal-1928727807-r6 { fill: #68a0b3;font-weight: bold } +.terminal-1928727807-r7 { fill: #868887 } +.terminal-1928727807-r8 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -135,37 +132,37 @@ - - - - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                     licences.py:77 -INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 -         packaged using conda.                                                         -INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + + + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                     licences.py:77 +INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 +         packaged using conda.                                                         +INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index ff0b82569c..6aecf73c89 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.1 - https://nf-co.re - - - -INFO     Testing pipeline: .__init__.py:263 - - -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check - - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━2 of 3 » fastqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc - - -╭─[?] 1 Pipeline Test Ignored ────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 5 Module Test Warnings ─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/modules…│New version available│ -│fastqc│modules/nf-core/modules…│New version available│ -│multiqc│modules/nf-core/modules…│New version available│ -│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ -│samplesheet_check│modules/local/sampleshe…│when: condition has been │ -│││removed│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 171 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   5 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Testing pipeline: .__init__.py:263 + + +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check +Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check + + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ +│samplesheet_check│modules/local/sampleshe…│when: condition has been │ +│││removed│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 177 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   2 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 34214e27c8..96d6e44f1c 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,104 +19,101 @@ font-weight: 700; } - .terminal-461755092-matrix { + .terminal-1481898385-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-461755092-title { + .terminal-1481898385-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-461755092-r1 { fill: #c5c8c6 } -.terminal-461755092-r2 { fill: #98a84b } -.terminal-461755092-r3 { fill: #9a9b99 } -.terminal-461755092-r4 { fill: #608ab1 } -.terminal-461755092-r5 { fill: #d0b344 } -.terminal-461755092-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-461755092-r7 { fill: #868887 } + .terminal-1481898385-r1 { fill: #c5c8c6 } +.terminal-1481898385-r2 { fill: #98a84b } +.terminal-1481898385-r3 { fill: #9a9b99 } +.terminal-1481898385-r4 { fill: #608ab1 } +.terminal-1481898385-r5 { fill: #d0b344 } +.terminal-1481898385-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1481898385-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -125,34 +122,34 @@ - - - - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnafusion            │    76 │          2.1.0 │ 2 months ago │           - │ -                   │ -│ smrnaseq             │    42 │          2.0.0 │ 3 months ago │           - │ -                   │ -│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ -│ dualrnaseq           │     7 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    19 │            dev │            - │           - │ -                   │ -│ lncpipe              │    23 │            dev │            - │           - │ -                   │ -│ scflow               │    12 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    10 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + + + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   510 │            3.9 │   3 days ago │           - │ -                   │ +│ rnafusion            │    80 │          2.1.0 │ 3 months ago │           - │ -                   │ +│ smrnaseq             │    44 │          2.0.0 │ 4 months ago │           - │ -                   │ +│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    20 │            dev │            - │           - │ -                   │ +│ lncpipe              │    22 │            dev │            - │           - │ -                   │ +│ scflow               │    13 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 70de3d250e..7ac91ab58d 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-478910217-matrix { + .terminal-2763061958-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-478910217-title { + .terminal-2763061958-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-478910217-r1 { fill: #c5c8c6 } -.terminal-478910217-r2 { fill: #98a84b } -.terminal-478910217-r3 { fill: #9a9b99 } -.terminal-478910217-r4 { fill: #608ab1 } -.terminal-478910217-r5 { fill: #d0b344 } -.terminal-478910217-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-478910217-r7 { fill: #868887 } -.terminal-478910217-r8 { fill: #868887;font-style: italic; } + .terminal-2763061958-r1 { fill: #c5c8c6 } +.terminal-2763061958-r2 { fill: #98a84b } +.terminal-2763061958-r3 { fill: #9a9b99 } +.terminal-2763061958-r4 { fill: #608ab1 } +.terminal-2763061958-r5 { fill: #d0b344 } +.terminal-2763061958-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2763061958-r7 { fill: #868887 } +.terminal-2763061958-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -111,30 +111,30 @@ - - - - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   501 │          3.8.1 │ 3 months ago │           - │ -                   │ -│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ -│ chipseq              │   122 │          1.2.2 │  1 years ago │           - │ -                   │ -│ atacseq              │   116 │          1.2.2 │ 4 months ago │           - │ -                   │ -[..truncated..] + + + + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   510 │            3.9 │   3 days ago │           - │ -                   │ +│ sarek                │   193 │          3.0.2 │   1 week ago │           - │ -                   │ +│ chipseq              │   127 │          2.0.0 │ 19 hours ago │           - │ -                   │ +│ atacseq              │   117 │          1.2.2 │ 5 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 1dba278236..925093b126 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3459298259-matrix { + .terminal-1207596895-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3459298259-title { + .terminal-1207596895-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3459298259-r1 { fill: #c5c8c6 } -.terminal-3459298259-r2 { fill: #98a84b } -.terminal-3459298259-r3 { fill: #9a9b99 } -.terminal-3459298259-r4 { fill: #608ab1 } -.terminal-3459298259-r5 { fill: #d0b344 } -.terminal-3459298259-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3459298259-r7 { fill: #868887 } -.terminal-3459298259-r8 { fill: #868887;font-style: italic; } + .terminal-1207596895-r1 { fill: #c5c8c6 } +.terminal-1207596895-r2 { fill: #98a84b } +.terminal-1207596895-r3 { fill: #9a9b99 } +.terminal-1207596895-r4 { fill: #608ab1 } +.terminal-1207596895-r5 { fill: #d0b344 } +.terminal-1207596895-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1207596895-r7 { fill: #868887 } +.terminal-1207596895-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,31 +114,31 @@ - - - - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ mag                  │    99 │          2.2.1 │   5 days ago │           - │ -                   │ -│ sarek                │   186 │          3.0.1 │  2 weeks ago │           - │ -                   │ -│ epitopeprediction    │    22 │          2.1.0 │  4 weeks ago │           - │ -                   │ -│ eager                │    71 │          2.4.5 │  4 weeks ago │           - │ -                   │ -│ viralrecon           │    75 │            2.5 │ 2 months ago │           - │ -                   │ -[..truncated..] + + + + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ chipseq              │   127 │          2.0.0 │ 19 hours ago │           - │ -                   │ +│ rnaseq               │   510 │            3.9 │   3 days ago │           - │ -                   │ +│ isoseq               │     5 │          1.1.1 │   6 days ago │           - │ -                   │ +│ sarek                │   193 │          3.0.2 │   1 week ago │           - │ -                   │ +│ airrflow             │    19 │          2.3.0 │  2 weeks ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index b857a8ab13..ee5dd12b1b 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,95 +19,92 @@ font-weight: 700; } - .terminal-2048094464-matrix { + .terminal-578811831-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2048094464-title { + .terminal-578811831-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2048094464-r1 { fill: #c5c8c6 } -.terminal-2048094464-r2 { fill: #98a84b } -.terminal-2048094464-r3 { fill: #9a9b99 } -.terminal-2048094464-r4 { fill: #608ab1 } -.terminal-2048094464-r5 { fill: #d0b344 } -.terminal-2048094464-r6 { fill: #98a84b;font-weight: bold } -.terminal-2048094464-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-578811831-r1 { fill: #c5c8c6 } +.terminal-578811831-r2 { fill: #98a84b } +.terminal-578811831-r3 { fill: #9a9b99 } +.terminal-578811831-r4 { fill: #608ab1 } +.terminal-578811831-r5 { fill: #d0b344 } +.terminal-578811831-r6 { fill: #98a84b;font-weight: bold } +.terminal-578811831-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -116,31 +113,31 @@ - - - - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + + + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 0ed9272626..f9e2bc0c2d 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.1 - https://nf-co.re - - -INFO     Looking for test workflow entry points:                             test_yml_builder.py:123 -'tests/modules/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:157 -INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:325 -nextflow run ./tests/modules/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config  -c  -./tests/modules/fastqc/nextflow.config --outdir /tmp/tmpfzia90r4 --work-dir /tmp/tmpq2f6vtpl + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Looking for test workflow entry points:                             test_yml_builder.py:126 +'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:160 +INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:328 +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config  -c  +./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpk5ifvaf8 -work-dir /tmp/tmpfn1rbgwx diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index e2ede2a028..9057bbb9dd 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - - - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Repository type: modulescreate.py:93 -INFO    Press enter to use default values (shown in brackets)or type your own create.py:97 -responses. ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:165 -INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:191 -INFO     Using Singularity container:                                                  create.py:192 -'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' -INFO     Created / edited following files:                                             create.py:270 -           ./modules/fastqc/main.nf -           ./modules/fastqc/meta.yml -           ./tests/modules/fastqc/main.nf -           ./tests/modules/fastqc/test.yml -           ./tests/modules/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + + + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Repository type: modulescreate.py:96 +INFO    Press enter to use default values (shown in brackets)or type your own create.py:100 +responses. ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:168 +INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:194 +INFO     Using Singularity container:                                                  create.py:195 +'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' +INFO     Created / edited following files:                                             create.py:273 +           ./modules/nf-core/fastqc/main.nf +           ./modules/nf-core/fastqc/meta.yml +           ./tests/modules/nf-core/fastqc/main.nf +           ./tests/modules/nf-core/fastqc/test.yml +           ./tests/modules/nf-core/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index d73034af45..546a92b854 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,168 +19,165 @@ font-weight: 700; } - .terminal-2531955442-matrix { + .terminal-4091539959-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2531955442-title { + .terminal-4091539959-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2531955442-r1 { fill: #c5c8c6 } -.terminal-2531955442-r2 { fill: #98a84b } -.terminal-2531955442-r3 { fill: #9a9b99 } -.terminal-2531955442-r4 { fill: #608ab1 } -.terminal-2531955442-r5 { fill: #d0b344 } -.terminal-2531955442-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2531955442-r7 { fill: #98a84b;font-weight: bold } -.terminal-2531955442-r8 { fill: #868887 } -.terminal-2531955442-r9 { fill: #d08442 } -.terminal-2531955442-r10 { fill: #868887;font-style: italic; } -.terminal-2531955442-r11 { fill: #98729f } + .terminal-4091539959-r1 { fill: #c5c8c6 } +.terminal-4091539959-r2 { fill: #98a84b } +.terminal-4091539959-r3 { fill: #9a9b99 } +.terminal-4091539959-r4 { fill: #608ab1 } +.terminal-4091539959-r5 { fill: #d0b344 } +.terminal-4091539959-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4091539959-r7 { fill: #98a84b;font-weight: bold } +.terminal-4091539959-r8 { fill: #868887 } +.terminal-4091539959-r9 { fill: #d08442 } +.terminal-4091539959-r10 { fill: #868887;font-style: italic; } +.terminal-4091539959-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -189,54 +186,54 @@ - - + + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.1 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 0145ddcbe5..4237855a07 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - - - - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Installing 'abacas'install.py:116 -INFO     Include statement: include { ABACAS } from                                   install.py:125 -'../modules/nf-core/modules/abacas/main' + + + + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Installing 'abacas'install.py:128 +INFO     Include statement: include { ABACAS } from '../modules/nf-core/abacas/main'install.py:137 diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index dc7df7bf96..8125b33b9c 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:200 -INFO     Linting module: 'multiqc'__init__.py:204 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning ──────────────────────────────────────────────────────────────────────╮ -│                                           ╷                         ╷                            │ -│Module name                              │File path              │Test message              │ -│╶──────────────────────────────────────────┼─────────────────────────┼───────────────────────────╴│ -│multiqc│modules/multiqc/main.nf│Conda update: │ -│││bioconda::multiqc 1.10 -> │ -│││1.13a│ -│                                           ╵                         ╵                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:200 +INFO     Linting module: 'multiqc'__init__.py:204 + +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc +Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc + + +╭─[!] 1 Module Test Warning──────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│multiqc│modules/nf-core/multiqc…│Process label │ +│││(process_single) is not │ +│││among standard labels: │ +│││process_low,process_medi…│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔]  22 Tests Passed│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 04f83aacae..fb725ea51d 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-3046389154-matrix { + .terminal-2818080157-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3046389154-title { + .terminal-2818080157-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3046389154-r1 { fill: #c5c8c6 } -.terminal-3046389154-r2 { fill: #98a84b } -.terminal-3046389154-r3 { fill: #9a9b99 } -.terminal-3046389154-r4 { fill: #608ab1 } -.terminal-3046389154-r5 { fill: #d0b344 } -.terminal-3046389154-r6 { fill: #868887 } -.terminal-3046389154-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-3046389154-r8 { fill: #868887;font-style: italic; } + .terminal-2818080157-r1 { fill: #c5c8c6 } +.terminal-2818080157-r2 { fill: #98a84b } +.terminal-2818080157-r3 { fill: #9a9b99 } +.terminal-2818080157-r4 { fill: #608ab1 } +.terminal-2818080157-r5 { fill: #d0b344 } +.terminal-2818080157-r6 { fill: #868887 } +.terminal-2818080157-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2818080157-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,37 +132,37 @@ - - - - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                       list.py:124 - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name         ┃Repository     ┃Version SHA        ┃Message             ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftware… │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -│                      │                 │                     │ yaml files, add      │            │ -│                      │                 │                     │ yamllint config      │            │ -│                      │                 │                     │ (#1279)              │            │ -│ fastqc               │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -│                      │                 │                     │ yaml files, add      │            │ -│                      │                 │                     │ yamllint config      │            │ -│                      │                 │                     │ (#1279)              │            │ -│ multiqc              │ nf-core/modules │ e745e167c1020928ef… │ Fix formatting in    │ 2022-02-15 │ -[..truncated..] + + + + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                       list.py:136 + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 2662d214e3..e13e7b405b 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,110 @@ font-weight: 700; } - .terminal-1756799955-matrix { + .terminal-1267369783-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1756799955-title { + .terminal-1267369783-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1756799955-r1 { fill: #c5c8c6 } -.terminal-1756799955-r2 { fill: #98a84b } -.terminal-1756799955-r3 { fill: #9a9b99 } -.terminal-1756799955-r4 { fill: #608ab1 } -.terminal-1756799955-r5 { fill: #d0b344 } -.terminal-1756799955-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1756799955-r7 { fill: #868887 } -.terminal-1756799955-r8 { fill: #868887;font-style: italic; } + .terminal-1267369783-r1 { fill: #c5c8c6 } +.terminal-1267369783-r2 { fill: #98a84b } +.terminal-1267369783-r3 { fill: #9a9b99 } +.terminal-1267369783-r4 { fill: #608ab1 } +.terminal-1267369783-r5 { fill: #d0b344 } +.terminal-1267369783-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-1267369783-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1267369783-r8 { fill: #868887 } +.terminal-1267369783-r9 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,37 +133,37 @@ - - - - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Modules available from nf-core/modules (master):                                list.py:119 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + + + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):         list.py:131 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index 7b53700260..3581be2c97 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -19,75 +19,72 @@ font-weight: 700; } - .terminal-1435933291-matrix { + .terminal-670140257-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1435933291-title { + .terminal-670140257-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1435933291-r1 { fill: #c5c8c6 } -.terminal-1435933291-r2 { fill: #98a84b } -.terminal-1435933291-r3 { fill: #9a9b99 } -.terminal-1435933291-r4 { fill: #608ab1 } -.terminal-1435933291-r5 { fill: #d0b344 } -.terminal-1435933291-r6 { fill: #868887 } -.terminal-1435933291-r7 { fill: #00823d;font-weight: bold } -.terminal-1435933291-r8 { fill: #68a0b3;font-weight: bold } + .terminal-670140257-r1 { fill: #c5c8c6 } +.terminal-670140257-r2 { fill: #98a84b } +.terminal-670140257-r3 { fill: #9a9b99 } +.terminal-670140257-r4 { fill: #608ab1 } +.terminal-670140257-r5 { fill: #d0b344 } +.terminal-670140257-r6 { fill: #868887 } +.terminal-670140257-r7 { fill: #00823d;font-weight: bold } +.terminal-670140257-r8 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -96,24 +93,24 @@ - - - - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                              mulled.py:68 -INFO     Mulled container hash:                                                      __main__.py:828 -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + + + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                              mulled.py:68 +INFO     Mulled container hash:                                                      __main__.py:834 +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index c55c4bef64..77432e0e75 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Changes in module 'nf-core/modules/fastqc'modules_differ.py:252 -INFO    'modules/nf-core/modules/fastqc/meta.yml' is unchanged                modules_differ.py:257 -INFO     Changes in 'fastqc/main.nf':                                          modules_differ.py:266 - ---- modules/nf-core/modules/fastqc/main.nf -+++ modules/nf-core/modules/fastqc/main.nf -@@ -1,6 +1,6 @@ -process FASTQC {                                                                                   -    tag "$meta.id"                                                                                 --    label 'process_medium' -+    label 'process_low' - -    conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null)                                 -    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  - - -INFO     Patch file of 'nf-core/modules/fastqc' written to                              patch.py:115 -'modules/nf-core/modules/fastqc/fastqc.diff' + + + + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute           __main__.py:571 diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 82113c613b..9f14659a32 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,70 +19,67 @@ font-weight: 700; } - .terminal-4144914268-matrix { + .terminal-1501779057-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4144914268-title { + .terminal-1501779057-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4144914268-r1 { fill: #c5c8c6 } -.terminal-4144914268-r2 { fill: #98a84b } -.terminal-4144914268-r3 { fill: #9a9b99 } -.terminal-4144914268-r4 { fill: #608ab1 } -.terminal-4144914268-r5 { fill: #d0b344 } -.terminal-4144914268-r6 { fill: #868887 } + .terminal-1501779057-r1 { fill: #c5c8c6 } +.terminal-1501779057-r2 { fill: #98a84b } +.terminal-1501779057-r3 { fill: #9a9b99 } +.terminal-1501779057-r4 { fill: #608ab1 } +.terminal-1501779057-r5 { fill: #d0b344 } +.terminal-1501779057-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -91,23 +88,23 @@ - - - - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO     Removing abacas                                                                remove.py:52 + + + + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Removing abacas                                                                remove.py:53 diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 14ecee98fc..3ba96d4934 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,73 +19,70 @@ font-weight: 700; } - .terminal-3641449111-matrix { + .terminal-643775464-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3641449111-title { + .terminal-643775464-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3641449111-r1 { fill: #c5c8c6 } -.terminal-3641449111-r2 { fill: #98a84b } -.terminal-3641449111-r3 { fill: #9a9b99 } -.terminal-3641449111-r4 { fill: #608ab1 } -.terminal-3641449111-r5 { fill: #d0b344 } -.terminal-3641449111-r6 { fill: #868887 } + .terminal-643775464-r1 { fill: #c5c8c6 } +.terminal-643775464-r2 { fill: #98a84b } +.terminal-643775464-r3 { fill: #9a9b99 } +.terminal-643775464-r4 { fill: #608ab1 } +.terminal-643775464-r5 { fill: #d0b344 } +.terminal-643775464-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -94,24 +91,24 @@ - - - - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view'module_test.py:184 + + + + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view'module_test.py:178 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 754c3a5fab..e1db61bdfb 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,82 +19,79 @@ font-weight: 700; } - .terminal-2235813615-matrix { + .terminal-2441767763-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2235813615-title { + .terminal-2441767763-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2235813615-r1 { fill: #c5c8c6 } -.terminal-2235813615-r2 { fill: #98a84b } -.terminal-2235813615-r3 { fill: #9a9b99 } -.terminal-2235813615-r4 { fill: #608ab1 } -.terminal-2235813615-r5 { fill: #d0b344 } -.terminal-2235813615-r6 { fill: #868887 } + .terminal-2441767763-r1 { fill: #c5c8c6 } +.terminal-2441767763-r2 { fill: #98a84b } +.terminal-2441767763-r3 { fill: #9a9b99 } +.terminal-2441767763-r4 { fill: #608ab1 } +.terminal-2441767763-r5 { fill: #d0b344 } +.terminal-2441767763-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -103,27 +100,27 @@ - - - - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - - -INFO    'nf-core/modules/abacas' is already up to date                                update.py:160 -INFO     Updating 'nf-core/modules/custom/dumpsoftwareversions'update.py:516 -INFO     Updating 'nf-core/modules/fastqc'update.py:516 -INFO     Updating 'nf-core/modules/multiqc'update.py:516 -INFO     Updates complete ✨                                                           update.py:242 + + + + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                update.py:162 +INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date           update.py:162 +INFO    'modules/nf-core/fastqc' is already up to date                                update.py:162 +INFO    'modules/nf-core/multiqc' is already up to date                               update.py:162 +INFO     Updates complete ✨                                                           update.py:244 diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 6ed1ed8ea6..db945fd020 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,77 +19,74 @@ font-weight: 700; } - .terminal-3651117881-matrix { + .terminal-520878211-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3651117881-title { + .terminal-520878211-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3651117881-r1 { fill: #c5c8c6 } -.terminal-3651117881-r2 { fill: #98a84b } -.terminal-3651117881-r3 { fill: #9a9b99 } -.terminal-3651117881-r4 { fill: #608ab1 } -.terminal-3651117881-r5 { fill: #d0b344 } -.terminal-3651117881-r6 { fill: #98a84b;font-weight: bold } -.terminal-3651117881-r7 { fill: #868887 } -.terminal-3651117881-r8 { fill: #868887;font-weight: bold } -.terminal-3651117881-r9 { fill: #4e707b;font-weight: bold } -.terminal-3651117881-r10 { fill: #68a0b3;font-weight: bold } + .terminal-520878211-r1 { fill: #c5c8c6 } +.terminal-520878211-r2 { fill: #98a84b } +.terminal-520878211-r3 { fill: #9a9b99 } +.terminal-520878211-r4 { fill: #608ab1 } +.terminal-520878211-r5 { fill: #d0b344 } +.terminal-520878211-r6 { fill: #98a84b;font-weight: bold } +.terminal-520878211-r7 { fill: #868887 } +.terminal-520878211-r8 { fill: #868887;font-weight: bold } +.terminal-520878211-r9 { fill: #4e707b;font-weight: bold } +.terminal-520878211-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -98,24 +95,24 @@ - - - - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 27 params)schema.py:95 -INFO     Writing schema with 28 params: './nextflow_schema.json'schema.py:173 + + + + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 30 params)schema.py:95 +INFO     Writing schema with 31 params: './nextflow_schema.json'schema.py:173 diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index 445b5052ec..179672a372 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,73 +19,70 @@ font-weight: 700; } - .terminal-3803351505-matrix { + .terminal-4182908040-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3803351505-title { + .terminal-4182908040-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3803351505-r1 { fill: #c5c8c6 } -.terminal-3803351505-r2 { fill: #98a84b } -.terminal-3803351505-r3 { fill: #9a9b99 } -.terminal-3803351505-r4 { fill: #608ab1 } -.terminal-3803351505-r5 { fill: #d0b344 } -.terminal-3803351505-r6 { fill: #98a84b;font-weight: bold } -.terminal-3803351505-r7 { fill: #868887 } -.terminal-3803351505-r8 { fill: #868887;font-weight: bold } -.terminal-3803351505-r9 { fill: #4e707b;font-weight: bold } + .terminal-4182908040-r1 { fill: #c5c8c6 } +.terminal-4182908040-r2 { fill: #98a84b } +.terminal-4182908040-r3 { fill: #9a9b99 } +.terminal-4182908040-r4 { fill: #608ab1 } +.terminal-4182908040-r5 { fill: #d0b344 } +.terminal-4182908040-r6 { fill: #98a84b;font-weight: bold } +.terminal-4182908040-r7 { fill: #868887 } +.terminal-4182908040-r8 { fill: #868887;font-weight: bold } +.terminal-4182908040-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -94,23 +91,23 @@ - - - - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 28 params)schema.py:95 + + + + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 31 params)schema.py:95 diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index c44b3174a7..755794c92b 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,76 +19,73 @@ font-weight: 700; } - .terminal-735030063-matrix { + .terminal-2556505363-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-735030063-title { + .terminal-2556505363-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-735030063-r1 { fill: #c5c8c6 } -.terminal-735030063-r2 { fill: #98a84b } -.terminal-735030063-r3 { fill: #9a9b99 } -.terminal-735030063-r4 { fill: #608ab1 } -.terminal-735030063-r5 { fill: #d0b344 } -.terminal-735030063-r6 { fill: #98a84b;font-weight: bold } -.terminal-735030063-r7 { fill: #868887 } -.terminal-735030063-r8 { fill: #868887;font-weight: bold } -.terminal-735030063-r9 { fill: #4e707b;font-weight: bold } + .terminal-2556505363-r1 { fill: #c5c8c6 } +.terminal-2556505363-r2 { fill: #98a84b } +.terminal-2556505363-r3 { fill: #9a9b99 } +.terminal-2556505363-r4 { fill: #608ab1 } +.terminal-2556505363-r5 { fill: #d0b344 } +.terminal-2556505363-r6 { fill: #98a84b;font-weight: bold } +.terminal-2556505363-r7 { fill: #868887 } +.terminal-2556505363-r8 { fill: #868887;font-weight: bold } +.terminal-2556505363-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -97,24 +94,24 @@ - - - - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 -INFO    [✓] Input parameters look validschema.py:213 + + + + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validationschema.py:237 +INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 +INFO    [✓] Input parameters look validschema.py:213 diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index d262bed244..dcd7d1790b 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,78 +19,75 @@ font-weight: 700; } - .terminal-3609966593-matrix { + .terminal-3476133111-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3609966593-title { + .terminal-3476133111-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3609966593-r1 { fill: #c5c8c6 } -.terminal-3609966593-r2 { fill: #98a84b } -.terminal-3609966593-r3 { fill: #9a9b99 } -.terminal-3609966593-r4 { fill: #608ab1 } -.terminal-3609966593-r5 { fill: #d0b344 } -.terminal-3609966593-r6 { fill: #98729f } -.terminal-3609966593-r7 { fill: #ff2c7a } -.terminal-3609966593-r8 { fill: #868887 } + .terminal-3476133111-r1 { fill: #c5c8c6 } +.terminal-3476133111-r2 { fill: #98a84b } +.terminal-3476133111-r3 { fill: #9a9b99 } +.terminal-3476133111-r4 { fill: #608ab1 } +.terminal-3476133111-r5 { fill: #d0b344 } +.terminal-3476133111-r6 { fill: #98729f } +.terminal-3476133111-r7 { fill: #ff2c7a } +.terminal-3476133111-r8 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -99,25 +96,25 @@ - - - - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.5 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 -INFO     Making a new template pipeline using pipeline variables                         sync.py:223 + + + + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 +INFO     Original pipeline repository branch is 'master'sync.py:149 +INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 +INFO     Making a new template pipeline using pipeline variables                         sync.py:223 From 5f7939cc7462f8ae69066f0268e9c134e8eaa405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 4 Oct 2022 13:12:15 +0200 Subject: [PATCH 177/854] Update CHANGELOG.md Co-authored-by: Phil Ewels --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7402436e1..01f1e1a72c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,11 +26,12 @@ - Schema: Remove `allOf` if no definition groups are left. - Use contextlib to temporarily change working directories ([#1819](https://github.com/nf-core/tools/pull/1819)) - More helpful error messages if `nf-core download` can't parse a singularity image download -- Modules: If something is wrong with the local repo cache, offer to delete it and try again ([#1850](https://github.com/nf-core/tools/issues/1850)) -- Restructure code to work with the directory restructuring in [modules](https://github.com/nf-core/modules/pull/2141) ([#1859](https://github.com/nf-core/tools/pull/1859)) ### Modules +- If something is wrong with the local repo cache, offer to delete it and try again ([#1850](https://github.com/nf-core/tools/issues/1850)) +- Restructure code to work with the directory restructuring in [modules](https://github.com/nf-core/modules/pull/2141) ([#1859](https://github.com/nf-core/tools/pull/1859)) + ## [v2.5.1 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5.1) - [2022-08-31] - Patch release to fix black linting in pipelines ([#1789](https://github.com/nf-core/tools/pull/1789)) From c1210efa887f6b32f5c7ef794e049471b7c25ae9 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 4 Oct 2022 12:29:45 +0100 Subject: [PATCH 178/854] Make label:process_single default when creating a new module --- CHANGELOG.md | 1 + nf_core/modules/create.py | 2 +- nf_core/modules/lint/main_nf.py | 2 +- tests/test_modules.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01f1e1a72c..eec564e3df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - If something is wrong with the local repo cache, offer to delete it and try again ([#1850](https://github.com/nf-core/tools/issues/1850)) - Restructure code to work with the directory restructuring in [modules](https://github.com/nf-core/modules/pull/2141) ([#1859](https://github.com/nf-core/tools/pull/1859)) +- Make `label: process_single` default when creating a new module ## [v2.5.1 - Gold Otter Patch](https://github.com/nf-core/tools/releases/tag/2.5.1) - [2022-08-31] diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index e50d2b22fe..5602d79a86 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -216,7 +216,7 @@ def create(self): default=author_default, ) - process_label_defaults = ["process_low", "process_medium", "process_high", "process_long"] + process_label_defaults = ["process_single", "process_low", "process_medium", "process_high", "process_long"] if self.process_label is None: log.info( "Provide an appropriate resource label for the process, taken from the " diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index b642825db4..46dae14d04 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -235,7 +235,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("process_capitals", "Process name is not in capital letters", self.main_nf)) # Check that process labels are correct - correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] + correct_process_labels = ["process_single", "process_low", "process_medium", "process_high", "process_long"] process_label = [l for l in lines if l.lstrip().startswith("label")] if len(process_label) > 0: try: diff --git a/tests/test_modules.py b/tests/test_modules.py index 8fb9bab31b..a4e777a623 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -33,7 +33,7 @@ def create_modules_repo_dummy(tmp_dir): fh.writelines(["repository_type: modules", "\n"]) # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) module_create.create() return root_dir From 2d151531c8c31bf42e2018680f8afb52868139d8 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 4 Oct 2022 12:36:09 +0100 Subject: [PATCH 179/854] Update process_single in create scripts --- nf_core/modules/create.py | 2 +- tests/modules/create.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 5602d79a86..24221f589b 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -228,7 +228,7 @@ def create(self): "Process resource label:", choices=process_label_defaults, style=nf_core.utils.nfcore_question_style, - default="process_low", + default="process_single", ).unsafe_ask() if self.has_meta is None: diff --git a/tests/modules/create.py b/tests/modules/create.py index 1ea2d66fa7..db7a66fa5c 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -8,7 +8,7 @@ def test_modules_create_succeed(self): """Succeed at creating the TrimGalore! module""" module_create = nf_core.modules.ModuleCreate( - self.pipeline_dir, "trimgalore", "@author", "process_low", True, True, conda_name="trim-galore" + self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) module_create.create() assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) @@ -17,7 +17,7 @@ def test_modules_create_succeed(self): def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" module_create = nf_core.modules.ModuleCreate( - self.pipeline_dir, "trimgalore", "@author", "process_low", False, False, conda_name="trim-galore" + self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) module_create.create() with pytest.raises(UserWarning) as excinfo: From b051239cef5cd549514412738c00b1b343c44317 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 4 Oct 2022 12:36:38 +0100 Subject: [PATCH 180/854] Add quotes around all tags in test.yml --- nf_core/module-template/tests/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index b0eb645846..b77fc1ceeb 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -10,5 +10,5 @@ files: - path: "output/{{ tool }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: output/{{ tool }}/versions.yml + - path: "output/{{ tool }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b From 6b7bd2d2030f2bb36deed7e6ec5b96d7dd1491f1 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 4 Oct 2022 12:39:39 +0100 Subject: [PATCH 181/854] Add correct path to modules in pytest test.yml --- nf_core/module-template/tests/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index b77fc1ceeb..1d7881edc5 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,7 +1,7 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" - command: nextflow run ./tests/modules/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ tool_dir }}/nextflow.config + command: nextflow run ./tests/modules/nf-core/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/{{ tool_dir }}/nextflow.config tags: - "{{ tool }}" # {%- if subtool %} From 0e1a815f3ed2ed3ea61765c8a057778ee22ac6e9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 4 Oct 2022 14:38:13 +0200 Subject: [PATCH 182/854] Jinja2 templating in module test - remove placeholder comment lines --- nf_core/module-template/tests/test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 1d7881edc5..9fb8a88020 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -3,10 +3,8 @@ - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/nf-core/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/{{ tool_dir }}/nextflow.config tags: - - "{{ tool }}" - # {%- if subtool %} - - "{{ tool }}/{{ subtool }}" - # {%- endif %} + - "{{ tool }}{% if subtool -%}" + - "{{ tool }}/{{ subtool }}{%- endif %}" files: - path: "output/{{ tool }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc From f3998994f02e34f213ec413b7707dfca17ddfdff Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 4 Oct 2022 13:47:54 +0100 Subject: [PATCH 183/854] Remove extra space --- nf_core/modules/test_yml_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index d8733803d8..6369780ed6 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -169,7 +169,7 @@ def build_single_test(self, entry_point): while ep_test["command"] == "": # Don't think we need the last `-c` flag, but keeping to avoid having to update 100s modules. # See https://github.com/nf-core/tools/issues/1562 - default_val = f"nextflow run ./tests/modules/nf-core/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/{self.module_name}/nextflow.config" + default_val = f"nextflow run ./tests/modules/nf-core/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/{self.module_name}/nextflow.config" if self.no_prompts: ep_test["command"] = default_val else: From fb1c7e99c589b2c47ccc71e4b0eea43ce6a7791f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 15:06:51 +0200 Subject: [PATCH 184/854] create a function to check the modules folder structure --- nf_core/modules/modules_command.py | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 7e622b18a2..3197045590 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -147,3 +147,34 @@ def load_lint_config(self): self.lint_config = yaml.safe_load(fh) except FileNotFoundError: log.debug(f"No lint config file found: {config_fn}") + + def check_modules_structure(self, hide_progress): + """ + Check that the structure of the modules directory in a pipeline is the correct one: + 'modules/nf-core/TOOL/SUBTOOL + """ + if self.repo_type == "pipeline": + wrong_location_modules = [] + for directory, _, files in os.walk(Path(self.dir, "modules")): + if "main.nf" in files: + module_path = Path(directory).relative_to(Path(self.dir, "modules")) + parts = module_path.parts + # Check that there are modules installed directly under the 'modules' directory + if len(parts) <= 3 and parts[0] != self.modules_repo.repo_path and parts[0] != "local": + wrong_location_modules.append(module_path.parent) + # If there are modules installed in the wrong location + if len(wrong_location_modules) > 0: + # Remove the local copy of the modules repository + log.info(f"Removing '{self.modules_repo.local_repo_dir}'") + shutil.rmtree(self.modules_repo.local_repo_dir) + self.modules_repo.setup_local_repo( + self.modules_repo.remote_url, self.modules_repo.branch, hide_progress + ) + # Move wrong modules to the right directory + for module in wrong_location_modules: + correct_dir = Path("modules", self.modules_repo.repo_path, module) + wrong_dir = Path("modules", module) + wrong_dir.rename(correct_dir) + log.info(f"Moved {wrong_dir} to {correct_dir}.") + # Regenerate modules.json file + ModulesJson(self.dir).check_up_to_date() From b679e4a27f1a67257a5119c80814bbabacf7fc6f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 15:18:55 +0200 Subject: [PATCH 185/854] check modules directory in all modules commands --- nf_core/modules/bump_versions.py | 3 +++ nf_core/modules/create.py | 3 +++ nf_core/modules/info.py | 3 +++ nf_core/modules/install.py | 3 +++ nf_core/modules/list.py | 2 ++ nf_core/modules/module_test.py | 2 ++ nf_core/modules/modules_command.py | 4 ++-- nf_core/modules/patch.py | 3 +++ nf_core/modules/remove.py | 3 +++ nf_core/modules/test_yml_builder.py | 2 ++ nf_core/modules/update.py | 3 +++ 11 files changed, 29 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 3e318db418..6e08f47574 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -55,6 +55,9 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self.ignored = [] self.show_up_to_date = show_uptodate + # Check modules directory structure + self.check_modules_structure() + # Verify that this is not a pipeline self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) if not repo_type == "modules": diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index e50d2b22fe..8a97b34344 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -88,6 +88,9 @@ def create(self): and matching Docker / Singularity images from BioContainers. """ + # Check modules directory structure + self.check_modules_structure() + # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 6e4f0b2d3a..e5b45931c7 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -72,6 +72,9 @@ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): pipeline_dir = None if self.repo_type == "pipeline": + # Check modules directory structure + self.check_modules_structure() + # Check modules.json up to date self.modules_json = ModulesJson(self.dir) self.modules_json.check_up_to_date() else: diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 616963bbe8..b63dab0deb 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -37,6 +37,9 @@ def install(self, module): if not self.has_valid_directory(): return False + # Check modules directory structure + self.check_modules_structure() + # Verify that 'modules.json' is consistent with the installed modules modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index 81e7929058..c1fe890a7e 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -22,6 +22,8 @@ def list_modules(self, keywords=None, print_json=False): Get available module names from GitHub tree for repo and print as list to stdout """ + # Check modules directory structure + self.check_modules_structure() # Initialise rich table table = rich.table.Table() diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 107b0c9545..6448d741b7 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -76,6 +76,8 @@ def run(self): def _check_inputs(self): """Do more complex checks about supplied flags.""" + # Check modules directory structure + self.check_modules_structure() # Retrieving installed modules if self.repo_type == "modules": diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 3197045590..c2b6a1b118 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -148,7 +148,7 @@ def load_lint_config(self): except FileNotFoundError: log.debug(f"No lint config file found: {config_fn}") - def check_modules_structure(self, hide_progress): + def check_modules_structure(self): """ Check that the structure of the modules directory in a pipeline is the correct one: 'modules/nf-core/TOOL/SUBTOOL @@ -168,7 +168,7 @@ def check_modules_structure(self, hide_progress): log.info(f"Removing '{self.modules_repo.local_repo_dir}'") shutil.rmtree(self.modules_repo.local_repo_dir) self.modules_repo.setup_local_repo( - self.modules_repo.remote_url, self.modules_repo.branch, hide_progress + self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress ) # Move wrong modules to the right directory for module in wrong_location_modules: diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 890b0d4c49..ec5e05a277 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -33,6 +33,9 @@ def param_check(self, module): raise UserWarning(f"Module '{Path('modules', module_dir, module)}' does not exist in the pipeline") def patch(self, module=None): + # Check modules directory structure + self.check_modules_structure() + self.modules_json.check_up_to_date() self.param_check(module) modules = self.modules_json.get_all_modules()[self.modules_repo.remote_url] diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 22ce36d169..9f54efca37 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -21,6 +21,9 @@ def remove(self, module): log.error("You cannot remove a module in a clone of nf-core/modules") return False + # Check modules directory structure + self.check_modules_structure() + # Check whether pipeline is valid and with a modules.json file self.has_valid_directory() self.has_modules_file() diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index d8733803d8..0fda6b64ab 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -69,6 +69,8 @@ def run(self): def check_inputs(self): """Do more complex checks about supplied flags.""" + # Check modules directory structure + self.check_modules_structure() # Get the tool name if not specified if self.module_name is None: diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index e1dd2cc022..cd59a4109b 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -82,6 +82,9 @@ def update(self, module=None): self._parameter_checks() + # Check modules directory structure + self.check_modules_structure() + # Verify that 'modules.json' is consistent with the installed modules self.modules_json.check_up_to_date() From f33edf9d18e658515f1fc6f9665ba9e236989f76 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 15:23:18 +0200 Subject: [PATCH 186/854] fix modules dir check --- nf_core/modules/modules_command.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index c2b6a1b118..e641727151 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -160,7 +160,7 @@ def check_modules_structure(self): module_path = Path(directory).relative_to(Path(self.dir, "modules")) parts = module_path.parts # Check that there are modules installed directly under the 'modules' directory - if len(parts) <= 3 and parts[0] != self.modules_repo.repo_path and parts[0] != "local": + if parts[1] == "modules": wrong_location_modules.append(module_path.parent) # If there are modules installed in the wrong location if len(wrong_location_modules) > 0: @@ -172,8 +172,9 @@ def check_modules_structure(self): ) # Move wrong modules to the right directory for module in wrong_location_modules: - correct_dir = Path("modules", self.modules_repo.repo_path, module) - wrong_dir = Path("modules", module) + modules_dir = Path("modules").resolve() + correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(module.parts[2:])) + wrong_dir = Path(modules_dir, module) wrong_dir.rename(correct_dir) log.info(f"Moved {wrong_dir} to {correct_dir}.") # Regenerate modules.json file From d2983ddc2d8ec114ede6a7f7142922f8676b179c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 15:48:09 +0200 Subject: [PATCH 187/854] fix path to copy to --- nf_core/modules/modules_command.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index e641727151..578e9a0860 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -161,9 +161,10 @@ def check_modules_structure(self): parts = module_path.parts # Check that there are modules installed directly under the 'modules' directory if parts[1] == "modules": - wrong_location_modules.append(module_path.parent) + wrong_location_modules.append(module_path) # If there are modules installed in the wrong location if len(wrong_location_modules) > 0: + log.info("The modules folder structure is outdated. Reinstalling modules.") # Remove the local copy of the modules repository log.info(f"Removing '{self.modules_repo.local_repo_dir}'") shutil.rmtree(self.modules_repo.local_repo_dir) @@ -173,9 +174,11 @@ def check_modules_structure(self): # Move wrong modules to the right directory for module in wrong_location_modules: modules_dir = Path("modules").resolve() - correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(module.parts[2:])) + correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) wrong_dir = Path(modules_dir, module) - wrong_dir.rename(correct_dir) + shutil.move(wrong_dir, correct_dir) log.info(f"Moved {wrong_dir} to {correct_dir}.") + shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file - ModulesJson(self.dir).check_up_to_date() + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() From 7e9ace4fc324c6c0b1e8b354270a9d300b932230 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 17:07:55 +0200 Subject: [PATCH 188/854] fix check_up_to_date function --- nf_core/__main__.py | 2 +- nf_core/modules/modules_json.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9df0b2ff97..30aa3f7edd 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -538,7 +538,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff): sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) - sys.exit(1) + raise # nf-core modules patch diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 397d6abcf3..32bb9ac6f6 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -104,7 +104,6 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): """ if repos is None: repos = {} - # Check if there are any nf-core modules installed if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): repos[nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE] = {} @@ -509,11 +508,13 @@ def check_up_to_date(self): repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) modules_with_repos = ( - (install_dir, str(dir.relative_to(install_dir))) + ( + nf_core.modules.module_utils.path_from_remote(repo_url), + str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))), + ) for dir in missing_from_modules_json - for repo_url, repo_content in repos.items() - for install_dir, modules in repo_content["modules"].items() - if nf_core.utils.is_relative_to(dir, install_dir) + for repo_url in repos + if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url)) ) repos_with_modules = {} From ead1a9301d038ba33602bbde8e4086ad132f9031 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 17:33:15 +0200 Subject: [PATCH 189/854] make branch No to False instead of None --- nf_core/modules/modules_json.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 32bb9ac6f6..59297352d3 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -236,7 +236,7 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo) if correct_commit_sha is None: log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.") - choices = [{"name": "No", "value": None}] + [ + choices = [{"name": "No", "value": False}] + [ {"name": branch, "value": branch} for branch in (available_branches - tried_branches) ] branch = questionary.select( @@ -244,7 +244,7 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - if branch is None: + if not branch: action = questionary.select( f"Module is untracked '{module}'. Please select what action to take", choices=[ @@ -384,6 +384,7 @@ def has_git_url_and_modules(self): elif ( not isinstance(repo_url, str) or repo_url == "" + or not repo_url.startswith("http") or not isinstance(repo_entry["modules"], dict) or repo_entry["modules"] == {} ): From fb66b45c4d3d5d6456b932d6444a428531226e0b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 17:42:29 +0200 Subject: [PATCH 190/854] improve warnings --- nf_core/modules/modules_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 59297352d3..07b6aef454 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -235,12 +235,12 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): else: correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo) if correct_commit_sha is None: - log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.") + log.info(f"Was unable to find matching module {module} files in the {modules_repo.branch} branch.") choices = [{"name": "No", "value": False}] + [ {"name": branch, "value": branch} for branch in (available_branches - tried_branches) ] branch = questionary.select( - "Was the modules installed from a different branch in the remote?", + f"Was the module '{module}' installed from a different branch in the remote?", choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -388,7 +388,7 @@ def has_git_url_and_modules(self): or not isinstance(repo_entry["modules"], dict) or repo_entry["modules"] == {} ): - log.warning(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules") + log.debug(f"modules.json entry {repo_entry} has non-string or empty entries for git_url or modules.") return False return True From 3401ea5e5d498d14a9d59e91e6ca36e362088a0d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 17:45:06 +0200 Subject: [PATCH 191/854] improve warnings --- nf_core/modules/modules_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 07b6aef454..644a83cc93 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -235,12 +235,12 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): else: correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo) if correct_commit_sha is None: - log.info(f"Was unable to find matching module {module} files in the {modules_repo.branch} branch.") + log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.") choices = [{"name": "No", "value": False}] + [ {"name": branch, "value": branch} for branch in (available_branches - tried_branches) ] branch = questionary.select( - f"Was the module '{module}' installed from a different branch in the remote?", + f"Was the module '{module}' installed from a different branch in the remote?\nSelect 'No' for a local module", choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() From d276aa32356f7c552125f00c4f2c18644e31c653 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 17:47:47 +0200 Subject: [PATCH 192/854] remove raise from main --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 30aa3f7edd..9df0b2ff97 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -538,7 +538,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff): sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) - raise + sys.exit(1) # nf-core modules patch From dd5eb6f270801a1ccae5b7c0ab592c0e4e5ad55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 4 Oct 2022 17:52:26 +0200 Subject: [PATCH 193/854] Update comments from code review Co-authored-by: Phil Ewels --- nf_core/modules/modules_command.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 578e9a0860..116cd1287e 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -151,7 +151,10 @@ def load_lint_config(self): def check_modules_structure(self): """ Check that the structure of the modules directory in a pipeline is the correct one: - 'modules/nf-core/TOOL/SUBTOOL + modules/nf-core/TOOL/SUBTOOL + + Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: + modules/nf-core/modules/TOOL/SUBTOOL """ if self.repo_type == "pipeline": wrong_location_modules = [] From af1edde769aa56139056c18b57bbe00e97f96179 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 18:31:52 +0200 Subject: [PATCH 194/854] add linting of modules directory --- nf_core/lint/__init__.py | 2 ++ nf_core/lint/modules_structure.py | 33 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 nf_core/lint/modules_structure.py diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 7e61de3ac1..e0622c54f9 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -172,6 +172,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .files_unchanged import files_unchanged from .merge_markers import merge_markers from .modules_json import modules_json + from .modules_structure import modules_structure from .multiqc_config import multiqc_config from .nextflow_config import nextflow_config from .pipeline_name_conventions import pipeline_name_conventions @@ -227,6 +228,7 @@ def _get_all_lint_tests(release_mode): "merge_markers", "modules_json", "multiqc_config", + "modules_structure", ] + (["version_consistency"] if release_mode else []) def _load(self): diff --git a/nf_core/lint/modules_structure.py b/nf_core/lint/modules_structure.py new file mode 100644 index 0000000000..830e0d1f16 --- /dev/null +++ b/nf_core/lint/modules_structure.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import logging +import os +from pathlib import Path + +log = logging.getLogger(__name__) + + +def modules_structure(self): + """ + Check that the structure of the modules directory in a pipeline is the correct one: + modules/nf-core/TOOL/SUBTOOL + + Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: + modules/nf-core/modules/TOOL/SUBTOOL + """ + wrong_location_modules = [] + for directory, _, files in os.walk(Path(self.wf_path, "modules")): + if "main.nf" in files: + module_path = Path(directory).relative_to(Path(self.wf_path, "modules")) + parts = module_path.parts + # Check that there are modules installed directly under the 'modules' directory + if parts[1] == "modules": + wrong_location_modules.append(module_path) + # If there are modules installed in the wrong location + failed = [] + passed = [] + if len(wrong_location_modules) > 0: + failed = ["modules directory structure is outdated. Should be 'modules/nf-core/TOOL/SUBTOOL'"] + else: + passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"] + return {"passed": passed, "warned": [], "failed": failed, "ignored": []} From 16d39097128cd08d646f7762bf520b2cd313e44e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 4 Oct 2022 20:42:02 +0200 Subject: [PATCH 195/854] Create new API docs pages Ran python make_lint_md.py in api docs dir --- docs/api/_src/module_lint_tests/module_changes.md | 5 +++++ docs/api/_src/module_lint_tests/module_patch.md | 5 +++++ docs/api/_src/module_lint_tests/module_tests.md | 5 +++++ docs/api/_src/module_lint_tests/module_version.md | 5 +++++ docs/api/_src/pipeline_lint_tests/modules_structure.md | 5 +++++ 5 files changed, 25 insertions(+) create mode 100644 docs/api/_src/module_lint_tests/module_changes.md create mode 100644 docs/api/_src/module_lint_tests/module_patch.md create mode 100644 docs/api/_src/module_lint_tests/module_tests.md create mode 100644 docs/api/_src/module_lint_tests/module_version.md create mode 100644 docs/api/_src/pipeline_lint_tests/modules_structure.md diff --git a/docs/api/_src/module_lint_tests/module_changes.md b/docs/api/_src/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..ce2d428ca9 --- /dev/null +++ b/docs/api/_src/module_lint_tests/module_changes.md @@ -0,0 +1,5 @@ +# module_changes + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.module_changes +``` diff --git a/docs/api/_src/module_lint_tests/module_patch.md b/docs/api/_src/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..dc98d97a6f --- /dev/null +++ b/docs/api/_src/module_lint_tests/module_patch.md @@ -0,0 +1,5 @@ +# module_patch + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.module_patch +``` diff --git a/docs/api/_src/module_lint_tests/module_tests.md b/docs/api/_src/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..d10934c0d6 --- /dev/null +++ b/docs/api/_src/module_lint_tests/module_tests.md @@ -0,0 +1,5 @@ +# module_tests + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.module_tests +``` diff --git a/docs/api/_src/module_lint_tests/module_version.md b/docs/api/_src/module_lint_tests/module_version.md new file mode 100644 index 0000000000..a088b4cc66 --- /dev/null +++ b/docs/api/_src/module_lint_tests/module_version.md @@ -0,0 +1,5 @@ +# module_version + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.module_version +``` diff --git a/docs/api/_src/pipeline_lint_tests/modules_structure.md b/docs/api/_src/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..faa39ca77f --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,5 @@ +# modules_structure + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.modules_structure +``` From 7ca39d80195b2d24b347e51c4f60e1444378765d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 4 Oct 2022 22:15:21 +0200 Subject: [PATCH 196/854] Add env variable HIDE_PROGRESS to hide progress bars --- .github/workflows/rich-codex.yml | 3 +-- nf_core/lint/__init__.py | 4 ++-- nf_core/modules/bump_versions.py | 3 ++- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/modules_repo.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 669c138f46..67ec8fb867 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -24,8 +24,7 @@ jobs: uses: ewels/rich-codex@v1 env: COLUMNS: 100 - NFCORE_LINT_HIDE_PROGRESS: true - NFCORE_MODULES_LINT_HIDE_PROGRESS: true + HIDE_PROGRESS: "true" with: commit_changes: "true" clean_img_paths: docs/images/*.svg diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 7e61de3ac1..1ec3b973b6 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -8,7 +8,7 @@ import datetime import json import logging -import re +import os import git import rich @@ -311,7 +311,7 @@ def _lint_pipeline(self): rich.progress.BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, - disable=self.hide_progress, + disable=self.hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, ) with self.progress_bar: lint_progress = self.progress_bar.add_task( diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 3e318db418..1c2e0f600a 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -8,7 +8,7 @@ import logging import re -from pathlib import Path +import os import questionary import rich @@ -101,6 +101,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): rich.progress.BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, + disable=os.environ.get("HIDE_PROGRESS", None) is not None, ) with progress_bar: bump_progress = progress_bar.add_task( diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 91961fde60..034e7b5f4b 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -266,7 +266,7 @@ def lint_modules(self, modules, local=False, fix_version=False): "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, console=console, - disable=self.hide_progress, + disable=self.hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, ) with progress_bar: lint_progress = progress_bar.add_task( diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index b21dd0b9d0..cde6d26044 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -161,7 +161,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True): rich.progress.BarColumn(bar_width=None), "[bold yellow]{task.fields[state]}", transient=True, - disable=hide_progress, + disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, ) with pbar: self.repo = git.Repo.clone_from( @@ -186,7 +186,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True): rich.progress.BarColumn(bar_width=None), "[bold yellow]{task.fields[state]}", transient=True, - disable=hide_progress, + disable=hide_progress or os.environ.get("HIDE_PROGRESS", None) is not None, ) with pbar: self.repo.remotes.origin.fetch( From 4d7c8ad0083a662dfea5ebfcebb594a04d37d65c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 4 Oct 2022 22:19:12 +0200 Subject: [PATCH 197/854] Fix isort --- nf_core/modules/bump_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 1c2e0f600a..cd62cf2b7d 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -7,8 +7,8 @@ from __future__ import print_function import logging -import re import os +import re import questionary import rich From e37f6ab0240c391862dac921aa86ffe4a3356f9d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 4 Oct 2022 23:08:57 +0200 Subject: [PATCH 198/854] try to find module sha with the old repo path structure --- nf_core/modules/modules_json.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 644a83cc93..d0b60adf3a 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -234,6 +234,11 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): correct_commit_sha = self.find_correct_commit_sha(module, temp_module_dir, modules_repo) else: correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo) + if correct_commit_sha is None: + # Check in the old path + correct_commit_sha = self.find_correct_commit_sha( + module, repo_path / "modules" / module, modules_repo + ) if correct_commit_sha is None: log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.") choices = [{"name": "No", "value": False}] + [ From 1a5fb43a0107d12ebfb13a8ca07fb66448c76f2f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 4 Oct 2022 21:48:30 +0000 Subject: [PATCH 199/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-lint.svg | 232 ++++++++------------ docs/images/nf-core-list-rna.svg | 118 +++++----- docs/images/nf-core-list-stars.svg | 104 ++++----- docs/images/nf-core-list.svg | 108 ++++----- docs/images/nf-core-modules-create-test.svg | 108 ++++----- docs/images/nf-core-modules-create.svg | 136 +++++------- docs/images/nf-core-modules-install.svg | 78 +++---- docs/images/nf-core-modules-lint.svg | 199 +++++------------ docs/images/nf-core-modules-list-local.svg | 132 +++++------ docs/images/nf-core-modules-list-remote.svg | 134 +++++------ docs/images/nf-core-modules-remove.svg | 72 +++--- docs/images/nf-core-modules-test.svg | 76 +++---- docs/images/nf-core-modules-update.svg | 88 ++++---- 13 files changed, 721 insertions(+), 864 deletions(-) diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 6aecf73c89..d5b62b7c01 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Testing pipeline: .__init__.py:263 - - -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » samplesheet_check -Linting local modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » samplesheet_check - - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━1 of 3 » custom/dumpsoftwareversions -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━3 of 3 » multiqc - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ -│samplesheet_check│modules/local/sampleshe…│when: condition has been │ -│││removed│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 177 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   2 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Testing pipeline: .__init__.py:265 + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ +│samplesheet_check│modules/local/sampleshe…│when: condition has been │ +│││removed│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 178 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   2 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 96d6e44f1c..00b4170ed2 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-1481898385-matrix { + .terminal-1720318354-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1481898385-title { + .terminal-1720318354-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1481898385-r1 { fill: #c5c8c6 } -.terminal-1481898385-r2 { fill: #98a84b } -.terminal-1481898385-r3 { fill: #9a9b99 } -.terminal-1481898385-r4 { fill: #608ab1 } -.terminal-1481898385-r5 { fill: #d0b344 } -.terminal-1481898385-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1481898385-r7 { fill: #868887 } + .terminal-1720318354-r1 { fill: #c5c8c6 } +.terminal-1720318354-r2 { fill: #98a84b } +.terminal-1720318354-r3 { fill: #9a9b99 } +.terminal-1720318354-r4 { fill: #608ab1 } +.terminal-1720318354-r5 { fill: #d0b344 } +.terminal-1720318354-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1720318354-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   510 │            3.9 │   3 days ago │           - │ -                   │ -│ rnafusion            │    80 │          2.1.0 │ 3 months ago │           - │ -                   │ -│ smrnaseq             │    44 │          2.0.0 │ 4 months ago │           - │ -                   │ -│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    20 │            dev │            - │           - │ -                   │ -│ lncpipe              │    22 │            dev │            - │           - │ -                   │ -│ scflow               │    13 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   510 │            3.9 │   4 days ago │           - │ -                   │ +│ rnafusion            │    80 │          2.1.0 │ 3 months ago │           - │ -                   │ +│ smrnaseq             │    44 │          2.0.0 │ 4 months ago │           - │ -                   │ +│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    20 │            dev │            - │           - │ -                   │ +│ lncpipe              │    22 │            dev │            - │           - │ -                   │ +│ scflow               │    13 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 7ac91ab58d..5c61a13a3a 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-2763061958-matrix { + .terminal-1833040706-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2763061958-title { + .terminal-1833040706-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2763061958-r1 { fill: #c5c8c6 } -.terminal-2763061958-r2 { fill: #98a84b } -.terminal-2763061958-r3 { fill: #9a9b99 } -.terminal-2763061958-r4 { fill: #608ab1 } -.terminal-2763061958-r5 { fill: #d0b344 } -.terminal-2763061958-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2763061958-r7 { fill: #868887 } -.terminal-2763061958-r8 { fill: #868887;font-style: italic; } + .terminal-1833040706-r1 { fill: #c5c8c6 } +.terminal-1833040706-r2 { fill: #98a84b } +.terminal-1833040706-r3 { fill: #9a9b99 } +.terminal-1833040706-r4 { fill: #608ab1 } +.terminal-1833040706-r5 { fill: #d0b344 } +.terminal-1833040706-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1833040706-r7 { fill: #868887 } +.terminal-1833040706-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   510 │            3.9 │   3 days ago │           - │ -                   │ -│ sarek                │   193 │          3.0.2 │   1 week ago │           - │ -                   │ -│ chipseq              │   127 │          2.0.0 │ 19 hours ago │           - │ -                   │ -│ atacseq              │   117 │          1.2.2 │ 5 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   510 │            3.9 │   4 days ago │           - │ -                   │ +│ sarek                │   193 │          3.0.2 │  1 weeks ago │           - │ -                   │ +│ chipseq              │   127 │          2.0.0 │    yesterday │           - │ -                   │ +│ atacseq              │   117 │          1.2.2 │ 5 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 925093b126..3c20bf2e71 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-1207596895-matrix { + .terminal-602830802-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1207596895-title { + .terminal-602830802-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1207596895-r1 { fill: #c5c8c6 } -.terminal-1207596895-r2 { fill: #98a84b } -.terminal-1207596895-r3 { fill: #9a9b99 } -.terminal-1207596895-r4 { fill: #608ab1 } -.terminal-1207596895-r5 { fill: #d0b344 } -.terminal-1207596895-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1207596895-r7 { fill: #868887 } -.terminal-1207596895-r8 { fill: #868887;font-style: italic; } + .terminal-602830802-r1 { fill: #c5c8c6 } +.terminal-602830802-r2 { fill: #98a84b } +.terminal-602830802-r3 { fill: #9a9b99 } +.terminal-602830802-r4 { fill: #608ab1 } +.terminal-602830802-r5 { fill: #d0b344 } +.terminal-602830802-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-602830802-r7 { fill: #868887 } +.terminal-602830802-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ chipseq              │   127 │          2.0.0 │ 19 hours ago │           - │ -                   │ -│ rnaseq               │   510 │            3.9 │   3 days ago │           - │ -                   │ -│ isoseq               │     5 │          1.1.1 │   6 days ago │           - │ -                   │ -│ sarek                │   193 │          3.0.2 │   1 week ago │           - │ -                   │ -│ airrflow             │    19 │          2.3.0 │  2 weeks ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ chipseq              │   127 │          2.0.0 │    yesterday │           - │ -                   │ +│ rnaseq               │   510 │            3.9 │   4 days ago │           - │ -                   │ +│ isoseq               │     5 │          1.1.1 │   1 week ago │           - │ -                   │ +│ sarek                │   193 │          3.0.2 │  1 weeks ago │           - │ -                   │ +│ airrflow             │    19 │          2.3.0 │  3 weeks ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index f9e2bc0c2d..8f05428006 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3317135900-matrix { + .terminal-4198334470-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3317135900-title { + .terminal-4198334470-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3317135900-r1 { fill: #c5c8c6 } -.terminal-3317135900-r2 { fill: #98a84b } -.terminal-3317135900-r3 { fill: #9a9b99 } -.terminal-3317135900-r4 { fill: #608ab1 } -.terminal-3317135900-r5 { fill: #d0b344 } -.terminal-3317135900-r6 { fill: #868887 } -.terminal-3317135900-r7 { fill: #ff2c7a } -.terminal-3317135900-r8 { fill: #98729f } + .terminal-4198334470-r1 { fill: #c5c8c6 } +.terminal-4198334470-r2 { fill: #98a84b } +.terminal-4198334470-r3 { fill: #9a9b99 } +.terminal-4198334470-r4 { fill: #608ab1 } +.terminal-4198334470-r5 { fill: #d0b344 } +.terminal-4198334470-r6 { fill: #868887 } +.terminal-4198334470-r7 { fill: #ff2c7a } +.terminal-4198334470-r8 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Looking for test workflow entry points:                             test_yml_builder.py:126 -'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc'test_yml_builder.py:160 -INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:328 -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config  -c  -./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpk5ifvaf8 -work-dir /tmp/tmpfn1rbgwx + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Looking for test workflow entry points:                             test_yml_builder.py:128 +'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc_single_end'test_yml_builder.py:162 +INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:330 +nextflow run ./tests/modules/nf-core/fastqc -entry  +test_fastqc_single_end -c ./tests/config/nextflow.config -c  +./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpha4zaib5 -work-dir /tmp/tmp1swft0ph diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 9057bbb9dd..b845061eb0 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Repository type: modulescreate.py:96 -INFO    Press enter to use default values (shown in brackets)or type your own create.py:100 -responses. ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:168 -INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1'create.py:194 -INFO     Using Singularity container:                                                  create.py:195 -'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' -INFO     Created / edited following files:                                             create.py:273 -           ./modules/nf-core/fastqc/main.nf -           ./modules/nf-core/fastqc/meta.yml -           ./tests/modules/nf-core/fastqc/main.nf -           ./tests/modules/nf-core/fastqc/test.yml -           ./tests/modules/nf-core/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Repository type: modulescreate.py:99 +INFO    Press enter to use default values (shown in brackets)or type your own create.py:103 +responses. ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:171 +INFO     Could not find a Docker/Singularity container (Unexpected response code `503` create.py:200 +         for                                                                            +https://api.biocontainers.pro/ga4gh/trs/v2/tools/fastqc/versions/fastqc-0.11. +9) diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 4237855a07..def3a19771 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,69 +19,69 @@ font-weight: 700; } - .terminal-4086127963-matrix { + .terminal-3402980675-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4086127963-title { + .terminal-3402980675-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4086127963-r1 { fill: #c5c8c6 } -.terminal-4086127963-r2 { fill: #98a84b } -.terminal-4086127963-r3 { fill: #9a9b99 } -.terminal-4086127963-r4 { fill: #608ab1 } -.terminal-4086127963-r5 { fill: #d0b344 } -.terminal-4086127963-r6 { fill: #868887 } -.terminal-4086127963-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-3402980675-r1 { fill: #c5c8c6 } +.terminal-3402980675-r2 { fill: #98a84b } +.terminal-3402980675-r3 { fill: #9a9b99 } +.terminal-3402980675-r4 { fill: #608ab1 } +.terminal-3402980675-r5 { fill: #d0b344 } +.terminal-3402980675-r6 { fill: #868887 } +.terminal-3402980675-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -93,23 +93,23 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Installing 'abacas'install.py:128 -INFO     Include statement: include { ABACAS } from '../modules/nf-core/abacas/main'install.py:137 + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Installing 'abacas'install.py:131 +INFO     Include statement: include { ABACAS } from '../modules/nf-core/abacas/main'install.py:140 diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 8125b33b9c..fdaf8e8bad 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:200 -INFO     Linting module: 'multiqc'__init__.py:204 - -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━0 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc -Linting nf-core modules━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━1 of 1 » multiqc - - -╭─[!] 1 Module Test Warning──────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│multiqc│modules/nf-core/multiqc…│Process label │ -│││(process_single) is not │ -│││among standard labels: │ -│││process_low,process_medi…│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔]  22 Tests Passed│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Linting modules repo: '.'__init__.py:200 +INFO     Linting module: 'multiqc'__init__.py:204 + +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  23 Tests Passed │ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index fb725ea51d..02bc53bd4c 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2818080157-matrix { + .terminal-3572137377-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2818080157-title { + .terminal-3572137377-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2818080157-r1 { fill: #c5c8c6 } -.terminal-2818080157-r2 { fill: #98a84b } -.terminal-2818080157-r3 { fill: #9a9b99 } -.terminal-2818080157-r4 { fill: #608ab1 } -.terminal-2818080157-r5 { fill: #d0b344 } -.terminal-2818080157-r6 { fill: #868887 } -.terminal-2818080157-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2818080157-r8 { fill: #868887;font-style: italic; } + .terminal-3572137377-r1 { fill: #c5c8c6 } +.terminal-3572137377-r2 { fill: #98a84b } +.terminal-3572137377-r3 { fill: #9a9b99 } +.terminal-3572137377-r4 { fill: #608ab1 } +.terminal-3572137377-r5 { fill: #d0b344 } +.terminal-3572137377-r6 { fill: #868887 } +.terminal-3572137377-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3572137377-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                       list.py:136 - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                       list.py:138 + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index e13e7b405b..2d43fab033 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,110 +19,110 @@ font-weight: 700; } - .terminal-1267369783-matrix { + .terminal-1649706811-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1267369783-title { + .terminal-1649706811-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1267369783-r1 { fill: #c5c8c6 } -.terminal-1267369783-r2 { fill: #98a84b } -.terminal-1267369783-r3 { fill: #9a9b99 } -.terminal-1267369783-r4 { fill: #608ab1 } -.terminal-1267369783-r5 { fill: #d0b344 } -.terminal-1267369783-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-1267369783-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-1267369783-r8 { fill: #868887 } -.terminal-1267369783-r9 { fill: #868887;font-style: italic; } + .terminal-1649706811-r1 { fill: #c5c8c6 } +.terminal-1649706811-r2 { fill: #98a84b } +.terminal-1649706811-r3 { fill: #9a9b99 } +.terminal-1649706811-r4 { fill: #608ab1 } +.terminal-1649706811-r5 { fill: #d0b344 } +.terminal-1649706811-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-1649706811-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1649706811-r8 { fill: #868887 } +.terminal-1649706811-r9 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -134,36 +134,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):         list.py:131 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):         list.py:133 + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                   │ +│ abricate/run                             │ +│ abricate/summary                         │ +│ adapterremoval                           │ +│ adapterremovalfixprefix                  │ +│ agrvate                                  │ +│ allelecounter                            │ +│ ampir                                    │ +│ amplify/predict                          │ +[..truncated..] diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 9f14659a32..68f27318a4 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-1501779057-matrix { + .terminal-1533629559-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1501779057-title { + .terminal-1533629559-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1501779057-r1 { fill: #c5c8c6 } -.terminal-1501779057-r2 { fill: #98a84b } -.terminal-1501779057-r3 { fill: #9a9b99 } -.terminal-1501779057-r4 { fill: #608ab1 } -.terminal-1501779057-r5 { fill: #d0b344 } -.terminal-1501779057-r6 { fill: #868887 } + .terminal-1533629559-r1 { fill: #c5c8c6 } +.terminal-1533629559-r2 { fill: #98a84b } +.terminal-1533629559-r3 { fill: #9a9b99 } +.terminal-1533629559-r4 { fill: #608ab1 } +.terminal-1533629559-r5 { fill: #d0b344 } +.terminal-1533629559-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Removing abacas                                                                remove.py:53 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO     Removing abacas                                                                remove.py:56 diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 3ba96d4934..d815fd6787 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-643775464-matrix { + .terminal-566836186-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-643775464-title { + .terminal-566836186-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-643775464-r1 { fill: #c5c8c6 } -.terminal-643775464-r2 { fill: #98a84b } -.terminal-643775464-r3 { fill: #9a9b99 } -.terminal-643775464-r4 { fill: #608ab1 } -.terminal-643775464-r5 { fill: #d0b344 } -.terminal-643775464-r6 { fill: #868887 } + .terminal-566836186-r1 { fill: #c5c8c6 } +.terminal-566836186-r2 { fill: #98a84b } +.terminal-566836186-r3 { fill: #9a9b99 } +.terminal-566836186-r4 { fill: #608ab1 } +.terminal-566836186-r5 { fill: #d0b344 } +.terminal-566836186-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,23 +92,23 @@ - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view'module_test.py:178 + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view'module_test.py:180 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index e1db61bdfb..641fc97ac7 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,77 +19,77 @@ font-weight: 700; } - .terminal-2441767763-matrix { + .terminal-542272369-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2441767763-title { + .terminal-542272369-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2441767763-r1 { fill: #c5c8c6 } -.terminal-2441767763-r2 { fill: #98a84b } -.terminal-2441767763-r3 { fill: #9a9b99 } -.terminal-2441767763-r4 { fill: #608ab1 } -.terminal-2441767763-r5 { fill: #d0b344 } -.terminal-2441767763-r6 { fill: #868887 } + .terminal-542272369-r1 { fill: #c5c8c6 } +.terminal-542272369-r2 { fill: #98a84b } +.terminal-542272369-r3 { fill: #9a9b99 } +.terminal-542272369-r4 { fill: #608ab1 } +.terminal-542272369-r5 { fill: #d0b344 } +.terminal-542272369-r6 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +101,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                update.py:162 -INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date           update.py:162 -INFO    'modules/nf-core/fastqc' is already up to date                                update.py:162 -INFO    'modules/nf-core/multiqc' is already up to date                               update.py:162 -INFO     Updates complete ✨                                                           update.py:244 + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.6 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                update.py:165 +INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date           update.py:165 +INFO    'modules/nf-core/fastqc' is already up to date                                update.py:165 +INFO    'modules/nf-core/multiqc' is already up to date                               update.py:165 +INFO     Updates complete ✨                                                           update.py:247 From 8a58e45df8a72dd89a41c81a1d305f004738803f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 4 Oct 2022 23:53:41 +0200 Subject: [PATCH 200/854] Bump tools to v2.7dev --- CHANGELOG.md | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eec564e3df..30aeaa4835 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # nf-core/tools: Changelog +## v2.7dev + +### Template + +### Linting + +### General + +### Modules + ## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] ### Template diff --git a/setup.py b/setup.py index b0ffe22f6d..81956e8eeb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.6" +version = "2.7dev" with open("README.md") as f: readme = f.read() From 88329046235fee09d24dfabecd55e449dcc68e1c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 5 Oct 2022 00:07:12 +0200 Subject: [PATCH 201/854] Tag the correct gitpod docker image name --- .github/workflows/push_dockerhub_release.yml | 2 +- CHANGELOG.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 71245244d8..f07d5675f8 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -38,5 +38,5 @@ jobs: docker tag nfcore/tools:latest nfcore/tools:${{ github.event.release.tag_name }} docker push nfcore/tools:${{ github.event.release.tag_name }} docker push nfcore/gitpod:latest - docker tag nfcore/gitpod:latest nfcore/tools:${{ github.event.release.tag_name }} + docker tag nfcore/gitpod:latest nfcore/gitpod:${{ github.event.release.tag_name }} docker push nfcore/gitpod:${{ github.event.release.tag_name }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 30aeaa4835..595766159b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### General +- Fix error in tagging GitPod docker images during releases + ### Modules ## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] From bc670f2125e83d70a2b8335d4aba130972154835 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 5 Oct 2022 06:43:52 +0100 Subject: [PATCH 202/854] Fix lint warnings for samplesheet_check.nf module --- CHANGELOG.md | 2 ++ nf_core/pipeline-template/modules/local/samplesheet_check.nf | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30aeaa4835..3be0c8e73d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Fix lint warnings for `samplesheet_check.nf` module + ### Linting ### General diff --git a/nf_core/pipeline-template/modules/local/samplesheet_check.nf b/nf_core/pipeline-template/modules/local/samplesheet_check.nf index ccfdfecc2c..03a50c1040 100644 --- a/nf_core/pipeline-template/modules/local/samplesheet_check.nf +++ b/nf_core/pipeline-template/modules/local/samplesheet_check.nf @@ -1,5 +1,6 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" + label 'process_single' conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -13,6 +14,9 @@ process SAMPLESHEET_CHECK { path '*.csv' , emit: csv path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: // This script is bundled with the pipeline, in {{ name }}/bin/ """ check_samplesheet.py \\ From 22357fbc2be22d05b7893459cc9e89051c484b5a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 5 Oct 2022 12:29:59 +0200 Subject: [PATCH 203/854] add a function to check if patch files contain outdated paths --- nf_core/modules/modules_command.py | 27 ++++++++++++++++++++++++++- nf_core/modules/patch.py | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 116cd1287e..ce0983eb0e 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -177,11 +177,36 @@ def check_modules_structure(self): # Move wrong modules to the right directory for module in wrong_location_modules: modules_dir = Path("modules").resolve() - correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) + module_name = str(Path(*module.parts[2:])) + correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(module_name)) wrong_dir = Path(modules_dir, module) shutil.move(wrong_dir, correct_dir) log.info(f"Moved {wrong_dir} to {correct_dir}.") + # Check if a path file exists + patch_path = correct_dir / Path(module_name + ".diff") + self.check_patch_paths(patch_path, module_name) shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() + + def check_patch_paths(self, patch_path, module_name): + """ + Check that paths in patch files are updated to the new modules path + """ + if patch_path.exists(): + rewrite = False + with open(patch_path, "r") as fh: + lines = fh.readlines() + for line in lines: + # Check if there are old paths in the patch file and replace + if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line: + rewrite = True + line.replace( + f"modules/{self.modules_repo.repo_path}/modules/{module_name}/", + f"modules/{self.modules_repo.repo_path}/{module_name}/", + ) + if rewrite: + with open(patch_path, "w") as fh: + for line in lines: + fh.write(line) diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index ec5e05a277..565a193729 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -50,7 +50,7 @@ def patch(self, module=None): module_dir = [dir for dir, m in modules if m == module][0] module_fullname = str(Path("modules", module_dir, module)) - # Verify that the module has an entry is the modules.json file + # Verify that the module has an entry in the modules.json file if not self.modules_json.module_present(module, self.modules_repo.remote_url, module_dir): raise UserWarning( f"The '{module_fullname}' module does not have an entry in the 'modules.json' file. Cannot compute patch" From ad6e31bf0e3ea2282190f78b8ed5ca433f7d6b14 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Wed, 5 Oct 2022 12:03:49 +0000 Subject: [PATCH 204/854] NF-Core Sync includes template.yaml as option Fixes #1879 --- CHANGELOG.md | 1 + nf_core/__main__.py | 5 +++-- nf_core/create.py | 1 + nf_core/sync.py | 5 +++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb6955ecb..2a59ccbfd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### General - Fix error in tagging GitPod docker images during releases +- `nf-core sync` now supports the template YAML file using `-t/--template-yaml`. ### Modules diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9df0b2ff97..461a901ec9 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1054,7 +1054,8 @@ def bump_version(new_version, dir, nextflow): @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") -def sync(dir, from_branch, pull_request, github_repository, username): +@click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") +def sync(dir, from_branch, pull_request, github_repository, username, template_yaml): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. @@ -1071,7 +1072,7 @@ def sync(dir, from_branch, pull_request, github_repository, username): nf_core.utils.is_pipeline_directory(dir) # Sync the given pipeline dir - sync_obj = nf_core.sync.PipelineSync(dir, from_branch, pull_request, github_repository, username) + sync_obj = nf_core.sync.PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) try: sync_obj.sync() except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: diff --git a/nf_core/create.py b/nf_core/create.py index 1cd4fecba0..8b028d90af 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -39,6 +39,7 @@ class PipelineCreate(object): force (bool): Overwrites a given workflow directory with the same name. Defaults to False. May the force be with you. outdir (str): Path to the local output directory. + template_yaml (str): Path to template.yml file for pipeline creation settings. """ def __init__( diff --git a/nf_core/sync.py b/nf_core/sync.py index a663f1c7a7..365d6d1371 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -52,6 +52,7 @@ class PipelineSync(object): required_config_vars (list): List of nextflow variables required to make template pipeline gh_username (str): GitHub username gh_repo (str): GitHub repository name + template_yaml (str): Path to template.yml file for pipeline creation settings. """ def __init__( @@ -61,6 +62,7 @@ def __init__( make_pr=False, gh_repo=None, gh_username=None, + template_yaml_path=None, ): """Initialise syncing object""" @@ -77,6 +79,8 @@ def __init__( self.gh_repo = gh_repo self.pr_url = "" + self.template_yaml_path = template_yaml_path + # Set up the API auth if supplied on the command line self.gh_api = nf_core.utils.gh_api self.gh_api.lazy_init() @@ -233,6 +237,7 @@ def make_template_pipeline(self): force=True, outdir=self.pipeline_dir, author=self.wf_config["manifest.author"].strip('"').strip("'"), + template_yaml_path=self.template_yaml_path, plain=True, ).init_pipeline() From 90712625993dc27912d15570e84f77a09f565588 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 5 Oct 2022 16:50:58 +0200 Subject: [PATCH 205/854] fix some bugs --- nf_core/modules/modules_command.py | 14 ++++++++++++-- nf_core/modules/update.py | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index ce0983eb0e..d942a98cd2 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -195,18 +195,28 @@ def check_patch_paths(self, patch_path, module_name): Check that paths in patch files are updated to the new modules path """ if patch_path.exists(): + log.info(f"Modules {module_name} contains a patch file.") rewrite = False with open(patch_path, "r") as fh: lines = fh.readlines() - for line in lines: + for index, line in enumerate(lines): # Check if there are old paths in the patch file and replace if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line: rewrite = True - line.replace( + lines[index] = line.replace( f"modules/{self.modules_repo.repo_path}/modules/{module_name}/", f"modules/{self.modules_repo.repo_path}/{module_name}/", ) if rewrite: + log.info(f"Updating paths in {patch_path}") with open(patch_path, "w") as fh: for line in lines: fh.write(line) + # Update path in modules.json if the file is in the correct format + modules_json = ModulesJson(self.dir) + modules_json.load() + if modules_json.has_git_url_and_modules(): + modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ + self.modules_repo.repo_path + ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) + modules_json.dump() diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index cd59a4109b..4372ec8d06 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -613,6 +613,10 @@ def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_i patch_path = Path(self.dir / patch_relpath) module_relpath = Path("modules", repo_path, module) + # Check that paths in patch file are updated + print(patch_path, module) + self.check_patch_paths(patch_path, module) + # Copy the installed files to a new temporary directory to save them for later use temp_dir = Path(tempfile.mkdtemp()) temp_module_dir = temp_dir / module From 30efa06dc36cf8bbdb0537904aa6852947958a7f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 5 Oct 2022 16:52:14 +0200 Subject: [PATCH 206/854] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb6955ecb..27b8a60096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ ### Modules +- Update patch file paths if the modules directory has the old structure ([#1878](https://github.com/nf-core/tools/pull/1878)) + ## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] ### Template From 1c5ad078e9933e4063297441d5345cc9532af0a3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 5 Oct 2022 16:58:40 +0200 Subject: [PATCH 207/854] do not remove local copy of modules repo --- CHANGELOG.md | 1 + nf_core/modules/modules_command.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb6955ecb..2e583e606b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### General - Fix error in tagging GitPod docker images during releases +- Don't remove local copy of modules repo, only update it with fetch ([#1879](https://github.com/nf-core/tools/pull/1879)) ### Modules diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 116cd1287e..1020158b78 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -169,8 +169,7 @@ def check_modules_structure(self): if len(wrong_location_modules) > 0: log.info("The modules folder structure is outdated. Reinstalling modules.") # Remove the local copy of the modules repository - log.info(f"Removing '{self.modules_repo.local_repo_dir}'") - shutil.rmtree(self.modules_repo.local_repo_dir) + log.info(f"Updating '{self.modules_repo.local_repo_dir}'") self.modules_repo.setup_local_repo( self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress ) From a89faba67d63615314e2af804b00784f4ff4e9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 5 Oct 2022 17:12:22 +0200 Subject: [PATCH 208/854] Update nf_core/modules/update.py Co-authored-by: Phil Ewels --- nf_core/modules/update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 4372ec8d06..f40b3743f9 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -614,7 +614,6 @@ def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_i module_relpath = Path("modules", repo_path, module) # Check that paths in patch file are updated - print(patch_path, module) self.check_patch_paths(patch_path, module) # Copy the installed files to a new temporary directory to save them for later use From 713987a123a4c5b369f95509310be4dcf18481d9 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Wed, 5 Oct 2022 16:06:01 +0000 Subject: [PATCH 209/854] nf-core sync supports template.yaml within repository - Cache template.yml file before running nf-core sync. - If syncing fails, reset to start --- nf_core/sync.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 365d6d1371..b3211da7dd 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -80,6 +80,10 @@ def __init__( self.pr_url = "" self.template_yaml_path = template_yaml_path + # Save contents of template.yml for using outside of git. + if self.template_yaml_path is not None: + with open(self.template_yaml_path, "r") as template_yaml: + self.template_yaml_cache = template_yaml.read() # Set up the API auth if supplied on the command line self.gh_api = nf_core.utils.gh_api @@ -208,7 +212,7 @@ def delete_template_branch_files(self): # Delete everything log.info("Deleting all files in 'TEMPLATE' branch") for the_file in os.listdir(self.pipeline_dir): - if the_file == ".git": + if the_file == ".git" or the_file == self.template_yaml_path: continue file_path = os.path.join(self.pipeline_dir, the_file) log.debug(f"Deleting {file_path}") @@ -229,17 +233,29 @@ def make_template_pipeline(self): # Only show error messages from pipeline creation logging.getLogger("nf_core.create").setLevel(logging.ERROR) - nf_core.create.PipelineCreate( - name=self.wf_config["manifest.name"].strip('"').strip("'"), - description=self.wf_config["manifest.description"].strip('"').strip("'"), - version=self.wf_config["manifest.version"].strip('"').strip("'"), - no_git=True, - force=True, - outdir=self.pipeline_dir, - author=self.wf_config["manifest.author"].strip('"').strip("'"), - template_yaml_path=self.template_yaml_path, - plain=True, - ).init_pipeline() + # Re-write the template yaml from cache which may have been updated + if self.template_yaml_path and self.template_yaml_cache: + with open(self.template_yaml_path, "w") as template_path: + template_path.write(self.template_yaml_cache) + + try: + nf_core.create.PipelineCreate( + name=self.wf_config["manifest.name"].strip('"').strip("'"), + description=self.wf_config["manifest.description"].strip('"').strip("'"), + version=self.wf_config["manifest.version"].strip('"').strip("'"), + no_git=True, + force=True, + outdir=self.pipeline_dir, + author=self.wf_config["manifest.author"].strip('"').strip("'"), + template_yaml_path=self.template_yaml_path, + plain=True, + ).init_pipeline() + except Exception as err: + # If sync fails, remove template_yaml_path before raising error. + os.remove(self.template_yaml_path) + # Reset to where you were to prevent git getting messed up. + self.repo.git.reset("--hard") + raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}") def commit_template_changes(self): """If we have any changes with the new template files, make a git commit""" From 752fe61eb11bee7e8f46d0ab498fc12826203c9a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 6 Oct 2022 15:33:38 +0200 Subject: [PATCH 210/854] first try to add subworkflows create-test-yml --- nf_core/__main__.py | 29 ++ nf_core/modules/modules_json.py | 20 ++ nf_core/subworkflows/__init__.py | 1 + nf_core/subworkflows/test_yml_builder.py | 370 +++++++++++++++++++++++ 4 files changed, 420 insertions(+) create mode 100644 nf_core/subworkflows/test_yml_builder.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 12126f6386..87cf8ddc05 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -924,6 +924,35 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): sys.exit(1) +# nf-core subworkflows create-test-yml +@subworkflows.command("create-test-yml") +@click.pass_context +@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") +@click.option("-o", "--output", type=str, help="Path for output YAML file") +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output YAML file if it already exists") +@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") +def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): + """ + Auto-generate a test.yml file for a new subworkflow. + + Given the name of a module, runs the Nextflow test command and automatically generate + the required `test.yml` file based on the output files. + """ + try: + meta_builder = nf_core.subworkflows.SubworkflowsTestYmlBuilder( + module_name=subworkflow, + run_tests=run_tests, + test_yml_output_path=output, + force_overwrite=force, + no_prompts=no_prompts, + ) + meta_builder.run() + except (UserWarning, LookupError) as e: + log.critical(e) + sys.exit(1) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index d0b60adf3a..772b5843d4 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -36,6 +36,7 @@ def __init__(self, pipeline_dir): self.modules_dir = Path(self.dir, "modules") self.modules_json = None self.pipeline_modules = None + self.pipeline_subworkflows = None def create(self): """ @@ -815,3 +816,22 @@ def __str__(self): def __repr__(self): return self.__str__() + + def get_installed_subworkflows(self): + """ + Retrieves all pipeline subworkflows that are reported in the modules.json + + Returns: + (dict[str, [(str, str)]]): Dictionary indexed with the repo urls, with a + list of tuples (module_dir, subworkflow) as values + """ + if self.modules_json is None: + self.load() + if self.pipeline_subworkflows is None: + self.pipeline_subworkflows = {} + for repo, repo_entry in self.modules_json.get("repos", {}).items(): + if "subworkflows" in repo_entry: + for dir, subworkflow in repo_entry["subworkflows"].items(): + self.pipeline_subworkflows[repo] = [(dir, name) for name in subworkflow] + + return self.pipeline_modules diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index 7e2aef35c7..624c0960c5 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1 +1,2 @@ from .create import SubworkflowCreate +from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py new file mode 100644 index 0000000000..c59dcadc92 --- /dev/null +++ b/nf_core/subworkflows/test_yml_builder.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python +""" +The ModulesTestYmlBuilder class handles automatic generation of the modules test.yml file +along with running the tests and creating md5 sums +""" + +from __future__ import print_function + +import errno +import gzip +import hashlib +import io +import logging +import operator +import os +import re +import shlex +import subprocess +import tempfile +from pathlib import Path + +import questionary +import rich +import yaml +from rich.syntax import Syntax + +import nf_core.utils +from nf_core.modules.modules_json import ModulesJson + +# from .modules_command import ModuleCommand +from nf_core.modules.modules_repo import ModulesRepo + +log = logging.getLogger(__name__) + + +class SubworkflowsTestYmlBuilder(object): + def __init__( + self, + subworkflow=None, + directory=".", + run_tests=False, + test_yml_output_path=None, + force_overwrite=False, + no_prompts=False, + ): + # super().__init__(directory) + self.dir = directory + self.subworkflow = subworkflow + self.run_tests = run_tests + self.test_yml_output_path = test_yml_output_path + self.force_overwrite = force_overwrite + self.no_prompts = no_prompts + self.subworkflow_dir = None + self.subworkflow_test_main = None + self.entry_points = [] + self.tests = [] + self.errors = [] + self.modules_repo = ModulesRepo() + self.modules_json = ModulesJson(self.dir) + + def run(self): + """Run build steps""" + if not self.no_prompts: + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + ) + self.check_inputs() + self.scrape_workflow_entry_points() + self.build_all_tests() + self.print_test_yml() + if len(self.errors) > 0: + errors = "\n - ".join(self.errors) + raise UserWarning(f"Ran, but found errors:\n - {errors}") + + def check_inputs(self): + """Do more complex checks about supplied flags.""" + # Get the tool name if not specified + if self.subworkflow is None: + installed_subworkflows = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) + installed_subworkflows = [name for _, name in installed_subworkflows] + self.module_name = questionary.autocomplete( + "Subworkflow name:", + choices=installed_subworkflows, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + self.subworkflow_dir = os.path.join("subworkflows", self.modules_repo.repo_path, self.subworkflow) + self.subworkflow_test_main = os.path.join( + "tests", "subworkflows", self.modules_repo.repo_path, self.subworkflow, "main.nf" + ) + + # First, sanity check that the module directory exists + if not os.path.isdir(self.subworkflow_dir): + raise UserWarning(f"Cannot find directory '{self.subworkflow_dir}'.") + if not os.path.exists(self.subworkflow_test_main): + raise UserWarning(f"Cannot find module test workflow '{self.subworkflow_test_main}'") + + # Check that we're running tests if no prompts + if not self.run_tests and self.no_prompts: + log.debug("Setting run_tests to True as running without prompts") + self.run_tests = True + + # Get the output YAML file / check it does not already exist + while self.test_yml_output_path is None: + default_val = f"tests/subworkflows/{self.modules_repo.repo_path}/{self.subworkflow}/test.yml" + if self.no_prompts: + self.test_yml_output_path = default_val + else: + self.test_yml_output_path = rich.prompt.Prompt.ask( + "[violet]Test YAML output path[/] (- for stdout)", default=default_val + ).strip() + if self.test_yml_output_path == "": + self.test_yml_output_path = None + # Check that the output YAML file does not already exist + if ( + self.test_yml_output_path is not None + and self.test_yml_output_path != "-" + and os.path.exists(self.test_yml_output_path) + and not self.force_overwrite + ): + if rich.prompt.Confirm.ask( + f"[red]File exists! [green]'{self.test_yml_output_path}' [violet]Overwrite?" + ): + self.force_overwrite = True + else: + self.test_yml_output_path = None + if os.path.exists(self.test_yml_output_path) and not self.force_overwrite: + raise UserWarning( + f"Test YAML file already exists! '{self.test_yml_output_path}'. Use '--force' to overwrite." + ) + + def scrape_workflow_entry_points(self): + """Find the test workflow entry points from main.nf""" + log.info(f"Looking for test workflow entry points: '{self.subworkflow_test_main}'") + with open(self.subworkflow_test_main, "r") as fh: + for line in fh: + match = re.match(r"workflow\s+(\S+)\s+{", line) + if match: + self.entry_points.append(match.group(1)) + if len(self.entry_points) == 0: + raise UserWarning("No workflow entry points found in 'self.module_test_main'") + + def build_all_tests(self): + """ + Go over each entry point and build structure + """ + for entry_point in self.entry_points: + ep_test = self.build_single_test(entry_point) + if ep_test: + self.tests.append(ep_test) + + def build_single_test(self, entry_point): + """Given the supplied cli flags, prompt for any that are missing. + + Returns: Test command + """ + ep_test = { + "name": "", + "command": "", + "tags": [], + "files": [], + } + + # Print nice divider line + console = rich.console.Console() + console.print("[black]" + "─" * console.width) + + log.info(f"Building test meta for entry point '{entry_point}'") + + while ep_test["name"] == "": + default_val = f"{self.subworkflow} {entry_point}" + if self.no_prompts: + ep_test["name"] = default_val + else: + ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() + + while ep_test["command"] == "": + default_val = f"nextflow run ./tests/subworkflows/{self.modules_repo.repo_path}/{self.subworkflow} -entry {entry_point}" + if self.no_prompts: + ep_test["command"] = default_val + else: + ep_test["command"] = rich.prompt.Prompt.ask("[violet]Test command", default=default_val).strip() + + while len(ep_test["tags"]) == 0: + tag_defaults = [] + tag_defaults.append("subworkflows/" + self.subworkflow) + if self.no_prompts: + ep_test["tags"] = tag_defaults + else: + while len(ep_test["tags"]) == 0: + prompt_tags = rich.prompt.Prompt.ask( + "[violet]Test tags[/] (comma separated)", default=",".join(tag_defaults) + ).strip() + ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] + + ep_test["files"] = self.get_md5_sums(entry_point, ep_test["command"]) + + return ep_test + + def check_if_empty_file(self, fname): + """Check if the file is empty, or compressed empty""" + if os.path.getsize(fname) == 0: + return True + try: + with open(fname, "rb") as fh: + g_f = gzip.GzipFile(fileobj=fh, mode="rb") + if g_f.read() == b"": + return True + except Exception as e: + # Python 3.8+ + if hasattr(gzip, "BadGzipFile"): + if isinstance(e, gzip.BadGzipFile): + pass + # Python 3.7 + elif isinstance(e, OSError): + pass + else: + raise e + return False + + def _md5(self, fname): + """Generate md5 sum for file""" + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(io.DEFAULT_BUFFER_SIZE), b""): + hash_md5.update(chunk) + md5sum = hash_md5.hexdigest() + return md5sum + + def create_test_file_dict(self, results_dir, is_repeat=False): + """Walk through directory and collect md5 sums""" + test_files = [] + for root, _, files in os.walk(results_dir, followlinks=True): + for filename in files: + # Check that the file is not versions.yml + if filename == "versions.yml": + continue + file_path = os.path.join(root, filename) + # add the key here so that it comes first in the dict + test_file = {"path": file_path} + # Check that this isn't an empty file + if self.check_if_empty_file(file_path): + if not is_repeat: + self.errors.append(f"Empty file found! '{os.path.basename(file_path)}'") + # Add the md5 anyway, linting should fail later and can be manually removed if needed. + # Originally we skipped this if empty, but then it's too easy to miss the warning. + # Equally, if a file is legitimately empty we don't want to prevent this from working. + file_md5 = self._md5(file_path) + test_file["md5sum"] = file_md5 + # Switch out the results directory path with the expected 'output' directory + test_file["path"] = file_path.replace(results_dir, "output") + test_files.append(test_file) + + test_files = sorted(test_files, key=operator.itemgetter("path")) + + return test_files + + def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repeat=None): + """ + Recursively go through directories and subdirectories + and generate tuples of (, ) + returns: list of tuples + """ + + run_this_test = False + while results_dir is None: + if self.run_tests or run_this_test: + results_dir, results_dir_repeat = self.run_tests_workflow(command) + else: + results_dir = rich.prompt.Prompt.ask( + "[violet]Test output folder with results[/] (leave blank to run test)" + ) + if results_dir == "": + results_dir = None + run_this_test = True + elif not os.path.isdir(results_dir): + log.error(f"Directory '{results_dir}' does not exist") + results_dir = None + + test_files = self.create_test_file_dict(results_dir=results_dir) + + # If test was repeated, compare the md5 sums + if results_dir_repeat: + test_files_repeat = self.create_test_file_dict(results_dir=results_dir_repeat, is_repeat=True) + + # Compare both test.yml files + for i in range(len(test_files)): + if test_files[i].get("md5sum") and not test_files[i].get("md5sum") == test_files_repeat[i]["md5sum"]: + test_files[i].pop("md5sum") + test_files[i][ + "contains" + ] = "[ # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ]" + + if len(test_files) == 0: + raise UserWarning(f"Could not find any test result files in '{results_dir}'") + + return test_files + + def run_tests_workflow(self, command): + """Given a test workflow and an entry point, run the test workflow""" + + # The config expects $PROFILE and Nextflow fails if it's not set + if os.environ.get("PROFILE") is None: + os.environ["PROFILE"] = "" + if self.no_prompts: + log.info( + "Setting env var '$PROFILE' to an empty string as not set.\n" + "Tests will run with Docker by default. " + "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." + ) + else: + question = { + "type": "list", + "name": "profile", + "message": "Choose software profile", + "choices": ["Docker", "Singularity", "Conda"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + profile = answer["profile"].lower() + if profile in ["singularity", "conda"]: + os.environ["PROFILE"] = profile + log.info(f"Setting env var '$PROFILE' to '{profile}'") + + tmp_dir = tempfile.mkdtemp() + tmp_dir_repeat = tempfile.mkdtemp() + work_dir = tempfile.mkdtemp() + command_repeat = command + f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" + command += f" --outdir {tmp_dir} -work-dir {work_dir}" + + log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") + try: + nfconfig_raw = subprocess.check_output(shlex.split(command)) + log.info("Repeating test ...") + nfconfig_raw = subprocess.check_output(shlex.split(command_repeat)) + + except OSError as e: + if e.errno == errno.ENOENT and command.strip().startswith("nextflow "): + raise AssertionError( + "It looks like Nextflow is not installed. It is required for most nf-core functions." + ) + except subprocess.CalledProcessError as e: + output = rich.markup.escape(e.output.decode()) + raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{output}") + except Exception as e: + raise UserWarning(f"Error running test workflow: {e}") + else: + log.info("Test workflow finished!") + try: + log.debug(rich.markup.escape(nfconfig_raw)) + except TypeError: + log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) + + return tmp_dir, tmp_dir_repeat + + def print_test_yml(self): + """ + Generate the test yml file. + """ + + if self.test_yml_output_path == "-": + console = rich.console.Console() + yaml_str = yaml.dump(self.tests, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + console.print("\n", Syntax(yaml_str, "yaml"), "\n") + return + + try: + log.info(f"Writing to '{self.test_yml_output_path}'") + with open(self.test_yml_output_path, "w") as fh: + yaml.dump(self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + except FileNotFoundError as e: + raise UserWarning(f"Could not create test.yml file: '{e}'") From 9c5b35e13f9f2c7369b7892ab4455beef20bd04b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 6 Oct 2022 16:40:42 +0200 Subject: [PATCH 211/854] fix some bugs --- nf_core/__main__.py | 5 +++-- nf_core/modules/modules_json.py | 2 +- nf_core/modules/modules_repo.py | 19 +++++++++++++++++++ nf_core/subworkflows/test_yml_builder.py | 10 ++++------ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 87cf8ddc05..84969281cf 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -940,8 +940,8 @@ def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): the required `test.yml` file based on the output files. """ try: - meta_builder = nf_core.subworkflows.SubworkflowsTestYmlBuilder( - module_name=subworkflow, + meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( + subworkflow=subworkflow, run_tests=run_tests, test_yml_output_path=output, force_overwrite=force, @@ -950,6 +950,7 @@ def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): meta_builder.run() except (UserWarning, LookupError) as e: log.critical(e) + raise sys.exit(1) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 772b5843d4..ef57d9d9f4 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -834,4 +834,4 @@ def get_installed_subworkflows(self): for dir, subworkflow in repo_entry["subworkflows"].items(): self.pipeline_subworkflows[repo] = [(dir, name) for name in subworkflow] - return self.pipeline_modules + return self.pipeline_subworkflows diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index cde6d26044..44d69a8b83 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -138,6 +138,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa # Convenience variable self.modules_dir = os.path.join(self.local_repo_dir, "modules", self.repo_path) + self.subworkflows_dir = os.path.join(self.local_repo_dir, "subworkflows", self.repo_path) self.avail_module_names = None @@ -430,6 +431,24 @@ def get_avail_modules(self, checkout=True): ] return avail_module_names + def get_avail_subworkflows(self, checkout=True): + """ + Gets the names of the subworkflows in the repository. They are detected by + checking which directories have a 'main.nf' file + + Returns: + ([ str ]): The subworkflow names + """ + if checkout: + self.checkout_branch() + # Module directories are characterized by having a 'main.nf' file + avail_subworkflow_names = [ + os.path.relpath(dirpath, start=self.subworkflows_dir) + for dirpath, _, file_names in os.walk(self.subworkflows_dir) + if "main.nf" in file_names + ] + return avail_subworkflow_names + def get_meta_yml(self, module_name): """ Returns the contents of the 'meta.yml' file of a module diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index c59dcadc92..dbe020575a 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -33,7 +33,7 @@ log = logging.getLogger(__name__) -class SubworkflowsTestYmlBuilder(object): +class SubworkflowTestYmlBuilder(object): def __init__( self, subworkflow=None, @@ -76,11 +76,9 @@ def check_inputs(self): """Do more complex checks about supplied flags.""" # Get the tool name if not specified if self.subworkflow is None: - installed_subworkflows = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - installed_subworkflows = [name for _, name in installed_subworkflows] - self.module_name = questionary.autocomplete( + self.subworkflow = questionary.autocomplete( "Subworkflow name:", - choices=installed_subworkflows, + choices=self.modules_repo.get_avail_subworkflows(), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() self.subworkflow_dir = os.path.join("subworkflows", self.modules_repo.repo_path, self.subworkflow) @@ -326,7 +324,7 @@ def run_tests_workflow(self, command): command_repeat = command + f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" command += f" --outdir {tmp_dir} -work-dir {work_dir}" - log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") + log.info(f"Running '{self.subworkflow}' test with command:\n[violet]{command}") try: nfconfig_raw = subprocess.check_output(shlex.split(command)) log.info("Repeating test ...") From fdf72d4275e5ac0750fb8aa117ec5d037bc15802 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 6 Oct 2022 16:47:15 +0200 Subject: [PATCH 212/854] create-test-yml working but tags missing --- nf_core/subworkflows/test_yml_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index dbe020575a..afc2733c67 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -172,7 +172,7 @@ def build_single_test(self, entry_point): ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() while ep_test["command"] == "": - default_val = f"nextflow run ./tests/subworkflows/{self.modules_repo.repo_path}/{self.subworkflow} -entry {entry_point}" + default_val = f"nextflow run ./tests/subworkflows/{self.modules_repo.repo_path}/{self.subworkflow} -entry {entry_point} -c ./tests/config/nextflow.config" if self.no_prompts: ep_test["command"] = default_val else: From 88352307b3a6c74b999ab480bddd0e35940dedf9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 7 Oct 2022 10:18:40 +0200 Subject: [PATCH 213/854] create-test-yml adds tags recursively --- nf_core/subworkflows/test_yml_builder.py | 27 ++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index afc2733c67..3e284c2f30 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -181,12 +181,13 @@ def build_single_test(self, entry_point): while len(ep_test["tags"]) == 0: tag_defaults = [] tag_defaults.append("subworkflows/" + self.subworkflow) + tag_defaults += self.parse_module_tags() if self.no_prompts: - ep_test["tags"] = tag_defaults + ep_test["tags"] = sorted(tag_defaults) else: while len(ep_test["tags"]) == 0: prompt_tags = rich.prompt.Prompt.ask( - "[violet]Test tags[/] (comma separated)", default=",".join(tag_defaults) + "[violet]Test tags[/] (comma separated)", default=",".join(sorted(tag_defaults)) ).strip() ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] @@ -194,6 +195,28 @@ def build_single_test(self, entry_point): return ep_test + def parse_module_tags(self): + """ + Parse the subworkflow test main.nf file to retrieve all imported modules for adding tags. + """ + tags = [] + with open(Path(self.subworkflow_dir, "main.nf"), "r") as fh: + for line in fh: + regex = re.compile( + r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" + ) + match = regex.match(line) + if match and len(match.groups()) == 2: + name, link = match.groups() + if link.startswith("../../../"): + name_split = name.lower().split("_") + tags.append("/".join(name_split)) + if len(name_split) > 1: + tags.append(name_split[0]) + elif link.startswith("../"): + tags.append("subworkflows/" + name.lower()) + return list(set(tags)) + def check_if_empty_file(self, fname): """Check if the file is empty, or compressed empty""" if os.path.getsize(fname) == 0: From 2a84d3d4f5e05fd6267ccc7cd8de63402c08554b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 7 Oct 2022 13:39:17 +0200 Subject: [PATCH 214/854] small modifications of subworkflows template --- nf_core/subworkflow-template/subworkflows/main.nf | 1 - nf_core/subworkflow-template/tests/test.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/subworkflow-template/subworkflows/main.nf b/nf_core/subworkflow-template/subworkflows/main.nf index 7895f8aa97..dfebaa94fe 100644 --- a/nf_core/subworkflow-template/subworkflows/main.nf +++ b/nf_core/subworkflow-template/subworkflows/main.nf @@ -2,7 +2,6 @@ // https://github.com/nf-core/modules/tree/master/subworkflows // You can also ask for help via your pull request or on the #subworkflows channel on the nf-core Slack workspace: // https://nf-co.re/join -// TODO nf-core: A subworkflow SHOULD only import modules not other subworkflows // TODO nf-core: A subworkflow SHOULD import at least two modules include { SAMTOOLS_SORT } from '../../../modules/nf-core/samtools/sort/main' diff --git a/nf_core/subworkflow-template/tests/test.yml b/nf_core/subworkflow-template/tests/test.yml index 4462e7107a..8c64fe9254 100644 --- a/nf_core/subworkflow-template/tests/test.yml +++ b/nf_core/subworkflow-template/tests/test.yml @@ -2,7 +2,7 @@ # nf-core subworkflows create-test-yml {{ subworkflow_name_underscore }} - name: "{{ subworkflow_name }}" - command: nextflow run ./tests/subworkflows/nf-core/{{ subworkflow_dir }} -entry test_{{ subworkflow_name }} -c ./tests/config/nextflow.config -c ./tests/subworkflows/nf-core/{{ subworkflow_dir }}/nextflow.config + command: nextflow run ./tests/subworkflows/nf-core/{{ subworkflow_dir }} -entry test_{{ subworkflow_name }} -c ./tests/config/nextflow.config tags: - "{{ subworkflow_name }}" files: From 03f0e6b215a0ba59623bb429ef28e11065fc3a61 Mon Sep 17 00:00:00 2001 From: "Maxime U. Garcia" Date: Fri, 7 Oct 2022 14:58:03 +0200 Subject: [PATCH 215/854] Update pipeline.nf cf https://github.com/nf-core/rnaseq/issues/882 --- nf_core/pipeline-template/workflows/pipeline.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index fa9aa7d465..370cbb77c0 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -102,9 +102,9 @@ workflow {{ short_name|upper }} { MULTIQC ( ch_multiqc_files.collect(), - ch_multiqc_config.collect().ifEmpty([]), - ch_multiqc_custom_config.collect().ifEmpty([]), - ch_multiqc_logo.collect().ifEmpty([]) + ch_multiqc_config.toList(), + ch_multiqc_custom_config.toList(), + ch_multiqc_logo.toList() ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) From 39226f39b91cba2016607b2ff83f079031df0ecd Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 7 Oct 2022 16:35:48 +0200 Subject: [PATCH 216/854] add nf-core subworkflows install, recursive modules install missing --- nf_core/__main__.py | 47 +++++- nf_core/modules/modules_json.py | 54 +++++++ nf_core/modules/modules_repo.py | 82 ++++++++++ nf_core/subworkflows/__init__.py | 1 + nf_core/subworkflows/install.py | 192 +++++++++++++++++++++++ nf_core/subworkflows/test_yml_builder.py | 2 - 6 files changed, 373 insertions(+), 5 deletions(-) create mode 100644 nf_core/subworkflows/install.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 84969281cf..02e6c0f410 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -417,9 +417,9 @@ def subworkflows(ctx, git_remote, branch, no_pull): ctx.ensure_object(dict) # Place the arguments in a context object - ctx.obj["subworkflows_repo_url"] = git_remote - ctx.obj["subworkflows_repo_branch"] = branch - ctx.obj["subworkflows_repo_no_pull"] = no_pull + ctx.obj["modules_repo_url"] = git_remote + ctx.obj["modules_repo_branch"] = branch + ctx.obj["modules_repo_no_pull"] = no_pull # nf-core modules list subcommands @@ -954,6 +954,47 @@ def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): sys.exit(1) +# nf-core subworkflows install +@subworkflows.command() +@click.pass_context +@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", +) +@click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the subworkflow") +@click.option( + "-f", "--force", is_flag=True, default=False, help="Force reinstallation of subworkflow if it already exists" +) +@click.option("-s", "--sha", type=str, metavar="", help="Install subworkflow at commit SHA") +def install(ctx, subworkflow, dir, prompt, force, sha): + """ + Install DSL2 subworkflow within a pipeline. + + Fetches and installs subworkflow files from a remote repo e.g. nf-core/modules. + """ + try: + subworkflow_install = nf_core.subworkflows.SubworkflowInstall( + dir, + force, + prompt, + sha, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + exit_status = subworkflow_install.install(subworkflow) + if not exit_status and all: + sys.exit(1) + except (UserWarning, LookupError) as e: + log.error(e) + raise + sys.exit(1) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index ef57d9d9f4..f02bbccb81 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -588,6 +588,37 @@ def update(self, modules_repo, module_name, module_version, write_file=True): if write_file: self.dump() + def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version, write_file=True): + """ + Updates the 'module.json' file with new subworkflow info + + Args: + modules_repo (ModulesRepo): A ModulesRepo object configured for the new subworkflow + subworkflow_name (str): Name of new subworkflow + subworkflow_version (str): git SHA for the new subworkflow entry + write_file (bool): whether to write the updated modules.json to a file. + """ + if self.modules_json is None: + self.load() + repo_name = modules_repo.repo_path + remote_url = modules_repo.remote_url + branch = modules_repo.branch + if remote_url not in self.modules_json["repos"]: + self.modules_json["repos"][remote_url] = {"subworkflows": {repo_name: {}}} + if "subworkflows" not in self.modules_json["repos"][remote_url]: + # It's the first subworkflow installed in the pipeline! + self.modules_json["repos"][remote_url]["subworkflows"] = {repo_name: {}} + repo_subworkflows_entry = self.modules_json["repos"][remote_url]["subworkflows"][repo_name] + if subworkflow_name not in repo_subworkflows_entry: + repo_subworkflows_entry[subworkflow_name] = {} + repo_subworkflows_entry[subworkflow_name]["git_sha"] = subworkflow_version + repo_subworkflows_entry[subworkflow_name]["branch"] = branch + + # Sort the 'modules.json' repo entries + self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) + if write_file: + self.dump() + def remove_entry(self, module_name, repo_url, install_dir): """ Removes an entry from the 'modules.json' file. @@ -753,6 +784,29 @@ def get_module_version(self, module_name, repo_url, install_dir): .get("git_sha", None) ) + def get_subworkflow_version(self, subworkflow_name, repo_url, install_dir): + """ + Returns the version of a subworkflow + + Args: + subworkflow_name (str): Name of the module + repo_url (str): URL of the repository + install_dir (str): Name of the directory where subworkflows are installed + + Returns: + (str): The git SHA of the subworkflow if it exists, None otherwise + """ + if self.modules_json is None: + self.load() + return ( + self.modules_json.get("repos", {}) + .get(repo_url, {}) + .get("subworkflows", {}) + .get(install_dir, {}) + .get(subworkflow_name, {}) + .get("git_sha", None) + ) + def get_all_modules(self): """ Retrieves all pipeline modules that are reported in the modules.json diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 44d69a8b83..55a0d3412c 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -290,6 +290,18 @@ def module_exists(self, module_name, checkout=True): """ return module_name in self.get_avail_modules(checkout=checkout) + def subworkflow_exists(self, subworkflow_name, checkout=True): + """ + Check if a subworkflow exists in the branch of the repo + + Args: + subworkflow_name (str): The name of the subworkflow + + Returns: + (bool): Whether the subworkflow exists in this branch of the repository + """ + return subworkflow_name in self.get_avail_subworkflows(checkout=checkout) + def get_module_dir(self, module_name): """ Returns the file path of a module directory in the repo. @@ -302,6 +314,18 @@ def get_module_dir(self, module_name): """ return os.path.join(self.modules_dir, module_name) + def get_subworkflow_dir(self, subworkflow_name): + """ + Returns the file path of a subworkflow directory in the repo. + Does not verify that the path exists. + Args: + subworkflow_name (str): The name of the subworkflow + + Returns: + subworkflow_path (str): The path of the subworkflow in the local copy of the repository + """ + return os.path.join(self.subworkflows_dir, subworkflow_name) + def install_module(self, module_name, install_dir, commit): """ Install the module files into a pipeline at the given commit @@ -332,6 +356,36 @@ def install_module(self, module_name, install_dir, commit): self.checkout_branch() return True + def install_subworkflow(self, subworkflow_name, install_dir, commit): + """ + Install the subworkflow files into a pipeline at the given commit + + Args: + subworkflow_name (str): The name of the subworkflow + install_dir (str): The path where the subworkflow should be installed + commit (str): The git SHA for the version of the subworkflow to be installed + + Returns: + (bool): Whether the operation was successful or not + """ + # Check out the repository at the requested ref + try: + self.checkout(commit) + except git.GitCommandError: + return False + + # Check if the subworkflow exists in the branch + if not self.subworkflow_exists(subworkflow_name, checkout=False): + log.error(f"The requested subworkflow does not exists in the branch '{self.branch}' of {self.remote_url}'") + return False + + # Copy the files from the repo to the install folder + shutil.copytree(self.get_subworkflow_dir(subworkflow_name), Path(install_dir, subworkflow_name)) + + # Switch back to the tip of the branch + self.checkout_branch() + return True + def module_files_identical(self, module_name, base_path, commit): """ Checks whether the module files in a pipeline are identical to the ones in the remote @@ -380,12 +434,40 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) return commits + def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): + """ + Fetches the commit history the of requested subworkflow since a given date. The default value is + not arbitrary - it is the last time the structure of the nf-core/subworkflow repository was had an + update breaking backwards compatibility. + Args: + subworkflow_name (str): Name of subworkflow + modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question + per_page (int): Number of commits per page returned by API + page_nbr (int): Page number of the retrieved commits + since (str): Only show commits later than this timestamp. + Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. + + Returns: + ( dict ): Iterator of commit SHAs and associated (truncated) message + """ + self.checkout_branch() + subworkflow_path = os.path.join("subworkflows", self.repo_path, subworkflow_name) + commits = self.repo.iter_commits(max_count=depth, paths=subworkflow_path) + commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + return commits + def get_latest_module_version(self, module_name): """ Returns the latest commit in the repository """ return list(self.get_module_git_log(module_name, depth=1))[0]["git_sha"] + def get_latest_subworkflow_version(self, module_name): + """ + Returns the latest commit in the repository + """ + return list(self.get_subworkflow_git_log(module_name, depth=1))[0]["git_sha"] + def sha_exists_on_branch(self, sha): """ Verifies that a given commit sha exists on the branch diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index 624c0960c5..cad349893e 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,2 +1,3 @@ from .create import SubworkflowCreate +from .install import SubworkflowInstall from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py new file mode 100644 index 0000000000..1942a4fa7d --- /dev/null +++ b/nf_core/subworkflows/install.py @@ -0,0 +1,192 @@ +import logging +import os +import shutil + +import questionary + +import nf_core.modules.module_utils +import nf_core.utils +from nf_core.modules.modules_json import ModulesJson + +# from .modules_command import ModuleCommand +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, ModulesRepo + +log = logging.getLogger(__name__) + + +class SubworkflowInstall(object): + def __init__( + self, + pipeline_dir, + force=False, + prompt=False, + sha=None, + remote_url=None, + branch=None, + no_pull=False, + ): + # super().__init__(pipeline_dir, remote_url, branch, no_pull) + self.dir = pipeline_dir + self.force = force + self.prompt = prompt + self.sha = sha + self.modules_repo = ModulesRepo(remote_url, branch, no_pull) + try: + if self.dir: + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + else: + self.repo_type = None + except LookupError as e: + raise UserWarning(e) + + def install(self, subworkflow): + if self.repo_type == "modules": + log.error("You cannot install a subworkflow in a clone of nf-core/modules") + return False + # Check whether pipelines is valid + if not self.has_valid_directory(): + return False + + # Verify that 'modules.json' is consistent with the installed modules and subworkflows + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() # TODO: check subworkflows also!!!! + if "subworkflows" not in modules_json.modules_json["repos"][self.modules_repo.remote_url]: + # It's the first subworkflow installed in the pipeline! + modules_json.modules_json["repos"][self.modules_repo.remote_url]["subworkflows"] = { + self.modules_repo.repo_path: {} + } + + if self.prompt and self.sha is not None: + log.error("Cannot use '--sha' and '--prompt' at the same time!") + return False + + # Verify that the provided SHA exists in the repo + if self.sha: + if not self.modules_repo.sha_exists_on_branch(self.sha): + log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") + return False + + if subworkflow is None: + subworkflow = questionary.autocomplete( + "Subworkflow name:", + choices=self.modules_repo.get_avail_subworkflows(), + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Check that the supplied name is an available subworkflow + if subworkflow and subworkflow not in self.modules_repo.get_avail_subworkflows(): + log.error(f"Subworkflow '{subworkflow}' not found in list of available subworkflows.") + log.info("Use the command 'nf-core subworkflows list' to view available software") + return False + + if not self.modules_repo.subworkflow_exists(subworkflow): + warn_msg = f"Subworkflow '{subworkflow}' not found in remote '{self.modules_repo.remote_url}' ({self.modules_repo.branch})" + log.warning(warn_msg) + return False + + current_version = modules_json.get_subworkflow_version( + subworkflow, self.modules_repo.remote_url, self.modules_repo.repo_path + ) + + # Set the install folder based on the repository name + install_folder = os.path.join(self.dir, "subworkflows", self.modules_repo.repo_path) + + # Compute the subworkflow directory + subworkflow_dir = os.path.join(install_folder, subworkflow) + + # Check that the subworkflow is not already installed + if (current_version is not None and os.path.exists(subworkflow_dir)) and not self.force: + + log.error("Subworkflow is already installed.") + repo_flag = ( + "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " + ) + branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " + + log.info( + f"To update '{subworkflow}' run 'nf-core subworkflow {repo_flag}{branch_flag}update {subworkflow}'. To force reinstallation use '--force'" + ) + return False + + if self.sha: + version = self.sha + elif self.prompt: + try: + version = nf_core.modules.module_utils.prompt_module_version_sha( + subworkflow, + installed_sha=current_version, + modules_repo=self.modules_repo, + ) + except SystemError as e: + log.error(e) + return False + else: + # Fetch the latest commit for the subworkflow + version = self.modules_repo.get_latest_subworkflow_version(subworkflow) + + if self.force: + log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") + self.clear_subworkflow_dir(subworkflow, subworkflow_dir) + for repo_url, repo_content in modules_json.modules_json["repos"].items(): + for dir, dir_subworkflow in repo_content["subworkflows"].items(): + for name, _ in dir_subworkflow.items(): + if name == subworkflow and dir == self.modules_repo.repo_path: + repo_to_remove = repo_url + log.info( + f"Removing subworkflow '{self.modules_repo.repo_path}/{subworkflow}' from repo '{repo_to_remove}' from modules.json" + ) + modules_json.remove_entry(subworkflow, repo_to_remove, self.modules_repo.repo_path) + break + + log.info(f"{'Rei' if self.force else 'I'}nstalling '{subworkflow}'") + log.debug(f"Installing subworkflow '{subworkflow}' at hash {version} from {self.modules_repo.remote_url}") + + # Download subworkflow files + if not self.install_subworkflow_files(subworkflow, version, self.modules_repo, install_folder): + return False + + # Print include statement + subworkflow_name = subworkflow.upper() + log.info( + f"Include statement: include {{ {subworkflow_name} }} from '.{os.path.join(install_folder, subworkflow)}/main'" + ) + + # Update module.json with newly installed subworkflow + modules_json.update_subworkflow(self.modules_repo, subworkflow, version) + return True + + def has_valid_directory(self): + """Check that we were given a pipeline""" + if self.dir is None or not os.path.exists(self.dir): + log.error(f"Could not find pipeline: {self.dir}") + return False + main_nf = os.path.join(self.dir, "main.nf") + nf_config = os.path.join(self.dir, "nextflow.config") + if not os.path.exists(main_nf) and not os.path.exists(nf_config): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + return True + + def clear_subworkflow_dir(self, subworkflow_name, subworkflow_dir): + """Removes all files in the subworkflow directory""" + try: + shutil.rmtree(subworkflow_dir) + log.debug(f"Successfully removed {subworkflow_name} subworkflow") + return True + except OSError as e: + log.error(f"Could not remove subworkflow: {e}") + return False + + def install_subworkflow_files(self, subworkflow_name, subworkflow_version, modules_repo, install_dir): + """ + Installs a subworkflow into the given directory + + Args: + subworkflow_name (str): The name of the subworkflow + subworkflow_version (str): Git SHA for the version of the subworkflow to be installed + modules_repo (ModulesRepo): A correctly configured ModulesRepo object + install_dir (str): The path to where the subworkflow should be installed (should be the 'subworkflow/' dir of the pipeline) + + Returns: + (bool): Whether the operation was successful of not + """ + return modules_repo.install_subworkflow(subworkflow_name, install_dir, subworkflow_version) diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 3e284c2f30..c02b330bbd 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -26,8 +26,6 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson - -# from .modules_command import ModuleCommand from nf_core.modules.modules_repo import ModulesRepo log = logging.getLogger(__name__) From 86c7f7290a89cf686f4f2fb891c70fc8b083eca1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 7 Oct 2022 19:21:02 +0200 Subject: [PATCH 217/854] add new commands to command_groups --- nf_core/__main__.py | 6 +++++- nf_core/subworkflows/create.py | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 02e6c0f410..e2fc328b1a 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -56,9 +56,13 @@ }, ], "nf-core subworkflows": [ + { + "name": "For pipelines", + "commands": ["install"], + }, { "name": "Developing new subworkflows", - "commands": ["create"], + "commands": ["create", "create-test-yml"], }, ], } diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 9ab91fd2ac..e94e8d5f79 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -5,7 +5,6 @@ from __future__ import print_function -import glob import json import logging import os @@ -13,7 +12,6 @@ import subprocess import jinja2 -import questionary import rich import yaml from packaging.version import parse as parse_version From 0b653a8e7751993b39b0b8c4a6a4b491634a9caf Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 7 Oct 2022 20:28:49 +0200 Subject: [PATCH 218/854] try to fix subworkflows create --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 56005ebc33..384a7b5441 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,5 @@ include LICENSE include README.md graft nf_core/module-template graft nf_core/pipeline-template +graft nf_core/subworkflow-template include requirements.txt From 34b909ce3d3a41a0ad75b44e3cfe8e5f71488d4b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Sat, 8 Oct 2022 10:47:23 +0200 Subject: [PATCH 219/854] test.yml indentations fixed --- nf_core/__main__.py | 1 - nf_core/utils.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e2fc328b1a..1ebe5ece23 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -954,7 +954,6 @@ def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): meta_builder.run() except (UserWarning, LookupError) as e: log.critical(e) - raise sys.exit(1) diff --git a/nf_core/utils.py b/nf_core/utils.py index 9321ff9629..662ba2b0d8 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -745,12 +745,12 @@ def represent_dict_preserve_order(self, data): """ return self.represent_dict(data.items()) - def increase_indent(self, flow=False, indentless=False): + def increase_indent(self, flow=False): """Indent YAML lists so that YAML validates with Prettier See https://github.com/yaml/pyyaml/issues/234#issuecomment-765894586 """ - return super().increase_indent(flow=flow, indentless=indentless) + return super(CustomDumper, self).increase_indent(flow=flow, indentless=False) # HACK: insert blank lines between top-level objects # inspired by https://stackoverflow.com/a/44284819/3786245 From fc04467aa94d0cb8256a211a074a46193519a697 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 10:05:09 +0200 Subject: [PATCH 220/854] add subworkflows to names in pytests_modules.yml --- nf_core/subworkflows/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index e94e8d5f79..eb2f8276d9 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -129,7 +129,7 @@ def create(self): try: with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: pytest_modules_yml = yaml.safe_load(fh) - pytest_modules_yml[self.subworkflow] = [ + pytest_modules_yml["subworkflows/" + self.subworkflow] = [ f"subworkflows/nf-core/{self.subworkflow}/**", f"tests/subworkflows/nf-core/{self.subworkflow}/**", ] From ff9365038578025ce5935efd051244369430b697 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 10:08:19 +0200 Subject: [PATCH 221/854] fix indentless bug --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 662ba2b0d8..57907109f9 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -745,7 +745,7 @@ def represent_dict_preserve_order(self, data): """ return self.represent_dict(data.items()) - def increase_indent(self, flow=False): + def increase_indent(self, flow=False, indentless=False): """Indent YAML lists so that YAML validates with Prettier See https://github.com/yaml/pyyaml/issues/234#issuecomment-765894586 From fcff15cbc351f9209c222ac9d9182a207870c39e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 10:12:20 +0200 Subject: [PATCH 222/854] remove newline from test.yml template --- nf_core/subworkflow-template/tests/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/subworkflow-template/tests/test.yml b/nf_core/subworkflow-template/tests/test.yml index 8c64fe9254..f47686e335 100644 --- a/nf_core/subworkflow-template/tests/test.yml +++ b/nf_core/subworkflow-template/tests/test.yml @@ -1,6 +1,5 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core subworkflows create-test-yml {{ subworkflow_name_underscore }} - - name: "{{ subworkflow_name }}" command: nextflow run ./tests/subworkflows/nf-core/{{ subworkflow_dir }} -entry test_{{ subworkflow_name }} -c ./tests/config/nextflow.config tags: From 462da95b5a2e72dda82697a89f85a7d9a0b9a395 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 11:14:00 +0200 Subject: [PATCH 223/854] add subworkflows tag to subworkflows test.yml --- nf_core/subworkflow-template/tests/test.yml | 3 ++- nf_core/subworkflows/test_yml_builder.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/subworkflow-template/tests/test.yml b/nf_core/subworkflow-template/tests/test.yml index f47686e335..7393e88e38 100644 --- a/nf_core/subworkflow-template/tests/test.yml +++ b/nf_core/subworkflow-template/tests/test.yml @@ -3,7 +3,8 @@ - name: "{{ subworkflow_name }}" command: nextflow run ./tests/subworkflows/nf-core/{{ subworkflow_dir }} -entry test_{{ subworkflow_name }} -c ./tests/config/nextflow.config tags: - - "{{ subworkflow_name }}" + - "subworkflows" + - "subworkflows/{{ subworkflow_name }}" files: - path: "output/{{ subworkflow_name }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index c02b330bbd..c11172078d 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -177,7 +177,7 @@ def build_single_test(self, entry_point): ep_test["command"] = rich.prompt.Prompt.ask("[violet]Test command", default=default_val).strip() while len(ep_test["tags"]) == 0: - tag_defaults = [] + tag_defaults = ["subworkflows"] tag_defaults.append("subworkflows/" + self.subworkflow) tag_defaults += self.parse_module_tags() if self.no_prompts: From 6ba559388d8c4625160a541c735f42cb93217c86 Mon Sep 17 00:00:00 2001 From: Alan Hoyle Date: Mon, 10 Oct 2022 08:13:35 -0400 Subject: [PATCH 224/854] Fix URLs in module template The links to the nf-core github in the template --- nf_core/module-template/modules/main.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 8a61ea32d1..6cfd5af2d5 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,5 +1,5 @@ // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) -// https://github.com/nf-core/modules/tree/master/modules +// https://github.com/nf-core/modules/tree/master/modules/nf-core/ // You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: // https://nf-co.re/join // TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. @@ -32,7 +32,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" // MUST be provided as an input via a Groovy Map called "meta". // This information may not be required in some instances e.g. indexing reference genome files: - // https://github.com/nf-core/modules/blob/master/modules/bwa/index/main.nf + // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} @@ -53,7 +53,7 @@ process {{ tool_name_underscore|upper }} { {%- endif %} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified - // e.g. https://github.com/nf-core/modules/blob/master/modules/homer/annotatepeaks/main.nf + // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "task.ext.args" directive // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter From a5e3e22c51f00800f2b729d510ad4f0785558b6b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 15:21:47 +0200 Subject: [PATCH 225/854] update subworkflows install so it installs also imported modules and subworkflows --- nf_core/modules/install.py | 26 ++++++++++----- nf_core/subworkflows/install.py | 58 ++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index b63dab0deb..ec363f0c80 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -86,17 +86,25 @@ def install(self, module): # Check that the module is not already installed if (current_version is not None and os.path.exists(module_dir)) and not self.force: + log.info("Module is already installed.") + print(f"Module {module} is already installed.") - log.error("Module is already installed.") - repo_flag = ( - "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " - ) - branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " + self.force = questionary.confirm( + f"Module {module} is already installed. Do you want to force the reinstallation?", + style=nf_core.utils.nfcore_question_style, + default=False, + ).unsafe_ask() - log.info( - f"To update '{module}' run 'nf-core modules {repo_flag}{branch_flag}update {module}'. To force reinstallation use '--force'" - ) - return False + if not self.force: + repo_flag = ( + "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " + ) + branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " + + log.info( + f"To update '{module}' run 'nf-core modules {repo_flag}{branch_flag}update {module}'. To force reinstallation use '--force'" + ) + return False if self.sha: version = self.sha diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 1942a4fa7d..337453389b 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -1,11 +1,14 @@ import logging import os +import re import shutil +from pathlib import Path import questionary import nf_core.modules.module_utils import nf_core.utils +from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson # from .modules_command import ModuleCommand @@ -96,17 +99,25 @@ def install(self, subworkflow): # Check that the subworkflow is not already installed if (current_version is not None and os.path.exists(subworkflow_dir)) and not self.force: + log.info("Subworkflow is already installed.") + print(f"Subworkflow {subworkflow} is already installed.") - log.error("Subworkflow is already installed.") - repo_flag = ( - "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " - ) - branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " + self.force = questionary.confirm( + f"Subworkflow {subworkflow} is already installed.\nDo you want to force the reinstallation of this subworkflow and all it's imported modules?", + style=nf_core.utils.nfcore_question_style, + default=False, + ).unsafe_ask() - log.info( - f"To update '{subworkflow}' run 'nf-core subworkflow {repo_flag}{branch_flag}update {subworkflow}'. To force reinstallation use '--force'" - ) - return False + if not self.force: + repo_flag = ( + "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " + ) + branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " + + log.info( + f"To update '{subworkflow}' run 'nf-core subworkflow {repo_flag}{branch_flag}update {subworkflow}'. To force reinstallation use '--force'" + ) + return False if self.sha: version = self.sha @@ -145,6 +156,14 @@ def install(self, subworkflow): if not self.install_subworkflow_files(subworkflow, version, self.modules_repo, install_folder): return False + # Install included modules and subworkflows + modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) + for s_install in subworkflows_to_install: + self.install(s_install) + for m_install in modules_to_install: + module_install = ModuleInstall(self.dir, force=self.force, prompt=self.prompt) + module_install.install(m_install) + # Print include statement subworkflow_name = subworkflow.upper() log.info( @@ -190,3 +209,24 @@ def install_subworkflow_files(self, subworkflow_name, subworkflow_version, modul (bool): Whether the operation was successful of not """ return modules_repo.install_subworkflow(subworkflow_name, install_dir, subworkflow_version) + + def get_modules_subworkflows_to_install(self, subworkflow_dir): + """ + Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows. + """ + modules = [] + subworkflows = [] + with open(Path(subworkflow_dir, "main.nf"), "r") as fh: + for line in fh: + regex = re.compile( + r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" + ) + match = regex.match(line) + if match and len(match.groups()) == 2: + name, link = match.groups() + if link.startswith("../../../"): + name_split = name.lower().split("_") + modules.append("/".join(name_split)) + elif link.startswith("../"): + subworkflows.append(name.lower()) + return modules, subworkflows From 540c195f1cb3028b720fcb0794fef09a8d49ebbe Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 15:28:06 +0200 Subject: [PATCH 226/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b31fedd92..addfad36f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Fix error in tagging GitPod docker images during releases - Don't remove local copy of modules repo, only update it with fetch ([#1879](https://github.com/nf-core/tools/pull/1879)) +- Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) ### Modules From 1feb3905ed370e41c189e00bcb7795297b0a05bd Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 15:30:07 +0200 Subject: [PATCH 227/854] update changelog for previous PRs :D --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index addfad36f7..653c15201c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ ### General - Fix error in tagging GitPod docker images during releases -- Don't remove local copy of modules repo, only update it with fetch ([#1879](https://github.com/nf-core/tools/pull/1879)) +- Don't remove local copy of modules repo, only update it with fetch ([#1881](https://github.com/nf-core/tools/pull/1881)) +- Add subworkflow commands create-test-yml, create and install ([#1897](https://github.com/nf-core/tools/pull/1897)) - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) ### Modules From a19ce63f97a7031c47decf92776f44b531797a8d Mon Sep 17 00:00:00 2001 From: Alan Hoyle Date: Mon, 10 Oct 2022 09:37:50 -0400 Subject: [PATCH 228/854] Fixed BWA example link in create.py --- nf_core/modules/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 35ed295a57..f57685e7de 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -239,7 +239,7 @@ def create(self): "Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' " "MUST be provided as an input via a Groovy Map called 'meta'. " "This information may [italic]not[/] be required in some instances, for example " - "[link=https://github.com/nf-core/modules/blob/master/modules/bwa/index/main.nf]indexing reference genome files[/link]." + "[link=https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf]indexing reference genome files[/link]." ) while self.has_meta is None: self.has_meta = rich.prompt.Confirm.ask( From b56dbdbfd034b28b89ece4a034b852ca2987e015 Mon Sep 17 00:00:00 2001 From: jenmuell Date: Mon, 10 Oct 2022 15:52:15 +0200 Subject: [PATCH 229/854] fix #1293 --- nf_core/pipeline-template/docs/usage.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 2c9969d04a..afaceadbfe 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -33,7 +33,7 @@ CONTROL_REP1,AEG588A1_S1_L004_R1_001.fastq.gz,AEG588A1_S1_L004_R2_001.fastq.gz ### Full samplesheet -The pipeline will auto-detect whether a sample is single- or paired-end using the information provided in the samplesheet. The samplesheet can have as many columns as you desire, however, there is a strict requirement for the first 3 columns to match those defined in the table below. +The pipeline will auto-detect whether a sample is single- or paired-end using the information provided in the samplesheet. The samplesheet can have as many columns as you desire, however, there is a strict requirement for the first 3 columns to match those defined in the table below. A final samplesheet file consisting of both single- and paired-end data may look something like the one below. This is for 6 samples, where `TREATMENT_REP3` has been sequenced twice. @@ -87,9 +87,9 @@ nextflow pull {{ name }} It is a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. -First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. +First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest pipeline version - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. -This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. +This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. ## Core Nextflow arguments @@ -99,7 +99,7 @@ This version number will be logged in reports when you run the pipeline, so that Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. When using Biocontainers, most of these software packaging methods pull Docker containers from quay.io e.g [FastQC](https://quay.io/repository/biocontainers/fastqc) except for Singularity which directly downloads Singularity images via https hosted by the [Galaxy project](https://depot.galaxyproject.org/singularity/) and Conda which downloads and installs software locally from [Bioconda](https://bioconda.github.io/). +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -111,8 +111,11 @@ The pipeline also dynamically loads configurations from [https://github.com/nf-c Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. -If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended. +If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended, since it can lead to different results on different machines dependent on the computer enviroment. +- `test` + - A profile with a complete configuration for automated testing + - Includes links to test data so needs no other parameters - `docker` - A generic configuration profile to be used with [Docker](https://docker.com/) - `singularity` @@ -125,9 +128,7 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. -- `test` - - A profile with a complete configuration for automated testing - - Includes links to test data so needs no other parameters + ### `-resume` @@ -175,6 +176,11 @@ Work dir: Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` ``` +#### For beginners + +A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. + +#### Advanced option on process level To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/software/star/align/main.nf`. @@ -196,7 +202,7 @@ process { > > If you get a warning suggesting that the process selector isn't recognised check that the process name has been specified correctly. -### Updating containers +### Updating containers (advanced users) The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. If for some reason you need to use a different version of a particular tool with the pipeline then you just need to identify the `process` name and override the Nextflow `container` definition for that process using the `withName` declaration. For example, in the [nf-core/viralrecon](https://nf-co.re/viralrecon) pipeline a tool called [Pangolin](https://github.com/cov-lineages/pangolin) has been used during the COVID-19 pandemic to assign lineages to SARS-CoV-2 genome sequenced samples. Given that the lineage assignments change quite frequently it doesn't make sense to re-release the nf-core/viralrecon everytime a new version of Pangolin has been released. However, you can override the default container used by the pipeline by creating a custom config file and passing it as a command-line argument via `-c custom.config`. From e529ae04927ce36f38ebee9dd81a6f70fbc83b00 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 17:20:26 +0200 Subject: [PATCH 230/854] fix bug when updating modules from old version in old folder structure --- nf_core/modules/modules_repo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 55a0d3412c..2c03c9019b 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -432,6 +432,10 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 module_path = os.path.join("modules", self.repo_path, module_name) commits = self.repo.iter_commits(max_count=depth, paths=module_path) commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + # Grab commits also from previous modules structure + module_path = os.path.join("modules", module_name) + commits = self.repo.iter_commits(max_count=depth, paths=module_path) + commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) return commits def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): From ba36c76649f6042786599cbfce2f8b7b6db4ce4a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 17:20:52 +0200 Subject: [PATCH 231/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b31fedd92..6104a8e92b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Fix error in tagging GitPod docker images during releases - Don't remove local copy of modules repo, only update it with fetch ([#1879](https://github.com/nf-core/tools/pull/1879)) +- Fix bug when updating modules from old version in old folder structure ### Modules From e85549f2ba42ff1fc73d9bcab596c4b22c82e132 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 10 Oct 2022 17:34:04 +0200 Subject: [PATCH 232/854] fix a small mistake --- nf_core/modules/modules_repo.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 2c03c9019b..3db4040932 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -430,12 +430,17 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 """ self.checkout_branch() module_path = os.path.join("modules", self.repo_path, module_name) - commits = self.repo.iter_commits(max_count=depth, paths=module_path) - commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + commits_new = self.repo.iter_commits(max_count=depth, paths=module_path) + commits_new = [ + {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_new + ] # Grab commits also from previous modules structure module_path = os.path.join("modules", module_name) - commits = self.repo.iter_commits(max_count=depth, paths=module_path) - commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + commits_old = self.repo.iter_commits(max_count=depth, paths=module_path) + commits_old = [ + {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_old + ] + commits = commits_new + commits_old return commits def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): From fb46a2d4d351d64259f5237276f25b92b005e3c8 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:36:26 +0200 Subject: [PATCH 233/854] Add simple log statement if a config exists for the subworkflow to print the necessary include statement --- nf_core/modules/modules_repo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 55a0d3412c..4e45eab59f 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -381,6 +381,8 @@ def install_subworkflow(self, subworkflow_name, install_dir, commit): # Copy the files from the repo to the install folder shutil.copytree(self.get_subworkflow_dir(subworkflow_name), Path(install_dir, subworkflow_name)) + if Path(install_dir, subworkflow_name, 'nextflow.config').is_file(): + log.info(f"Subworkflow config include statement: includeConfig '{Path(install_dir, subworkflow_name, 'nextflow.config')}'") # Switch back to the tip of the branch self.checkout_branch() From 0c00bcefeb9c70a24ccf9fe5306a96ee7ddbaf1f Mon Sep 17 00:00:00 2001 From: Snorkan Date: Mon, 10 Oct 2022 18:05:00 +0200 Subject: [PATCH 234/854] Remove tick in the wrong place Removed tick that lead to wrong words were highlighted --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ad2846b23..c9548c6a40 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,7 @@ To help developers working with pipeline schema, nf-core tools has three `schema Nextflow can take input parameters in a JSON or YAML file when running a pipeline using the `-params-file` option. This command validates such a file against the pipeline schema. -`Usage is `nf-core schema validate `. eg with the pipeline downloaded [above](#download-pipeline), you can run: +Usage is `nf-core schema validate `. eg with the pipeline downloaded [above](#download-pipeline), you can run: - [`nf-core` tools installation](#installation) +- [`nf-core` tools update](#update-tools) - [`nf-core list` - List available pipelines](#listing-pipelines) - [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) - [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) @@ -186,6 +187,22 @@ If you would prefer to skip this check, set the environment variable `NFCORE_NO_ export NFCORE_NO_VERSION_CHECK=1 ``` +### Update tools + +It is advisable to keep nf-core/tools updated to the most recent version. The command to update depends on the system used to install it, for example if you have installed it with conda you can use: +```bash +conda update nf-core +``` +if you used pip: +```bash +pip install --upgrade nf-core +``` +or +```bash +pip install -U nf-core +``` +Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages). + ## Listing pipelines The command `nf-core list` shows all available nf-core pipelines along with their latest version, when that was published and how recently the pipeline code was pulled to your local system (if at all). From 31f1e69096d7128cb69529968d4386314ca173f4 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Tue, 11 Oct 2022 15:51:16 +0200 Subject: [PATCH 252/854] accommodate review --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index c30189eb4f..9160d7b231 100644 --- a/README.md +++ b/README.md @@ -196,11 +196,6 @@ conda update nf-core if you used pip: ```bash pip install --upgrade nf-core -``` -or -```bash -pip install -U nf-core -``` Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages). ## Listing pipelines From 518c9cb701233b26a089f3751a590ad09b8c50e9 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Tue, 11 Oct 2022 15:55:32 +0200 Subject: [PATCH 253/854] fix the code block markers --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9160d7b231..ce16f65a9c 100644 --- a/README.md +++ b/README.md @@ -193,9 +193,12 @@ It is advisable to keep nf-core/tools updated to the most recent version. The co ```bash conda update nf-core ``` + if you used pip: ```bash pip install --upgrade nf-core +``` + Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages). ## Listing pipelines From 8f414ad314a1af9e048306f89a0b0cf33f246b5c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 11 Oct 2022 16:17:58 +0200 Subject: [PATCH 254/854] check_up_to_date is also checking subworkflows --- nf_core/modules/modules_json.py | 176 +++++++++++++++++--------------- 1 file changed, 94 insertions(+), 82 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 918901a5d2..26cb755a43 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -455,14 +455,14 @@ def reinstall_repo(self, install_dir, remote_url, module_entries): def check_up_to_date(self): """ - Checks whether the modules installed in the directory + Checks whether the modules and subworkflows installed in the directory are consistent with the entries in the 'modules.json' file and vice versa. - If a module has an entry in the 'modules.json' file but is missing in the directory, - we first try to reinstall the module from the remote and if that fails we remove the entry + If a module/subworkflow has an entry in the 'modules.json' file but is missing in the directory, + we first try to reinstall the module/subworkflow from the remote and if that fails we remove the entry in 'modules.json'. - If a module is installed but the entry in 'modules.json' is missing we iterate through + If a module/subworkflow is installed but the entry in 'modules.json' is missing we iterate through the commit log in the remote to try to determine the SHA. """ try: @@ -479,88 +479,18 @@ def check_up_to_date(self): missing_installation, ) = self.unsynced_modules() - # If there are any modules left in 'modules.json' after all installed are removed, + # If there are any modules/subworkflows left in 'modules.json' after all installed are removed, # we try to reinstall them if len(missing_installation) > 0: - missing_but_in_mod_json = [ - f"'modules/{install_dir}/{module}'" - for repo_url, contents in missing_installation.items() - for install_dir, dir_contents in contents["modules"].items() - for module in dir_contents - ] - log.info( - f"Reinstalling modules found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" - ) - - remove_from_mod_json = {} - for repo_url, contents in missing_installation.items(): - for install_dir, module_entries in contents["modules"].items(): - remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( - install_dir, repo_url, module_entries - ) - - # If the reinstall fails, we remove those entries in 'modules.json' - if sum(map(len, remove_from_mod_json.values())) > 0: - uninstallable_mods = [ - f"'{install_dir}/{module}'" - for (repo_url, install_dir), modules in remove_from_mod_json.items() - for module in modules - ] - if len(uninstallable_mods) == 1: - log.info(f"Was unable to reinstall {uninstallable_mods[0]}. Removing 'modules.json' entry") - else: - log.info( - f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_mods)}" - ) - - for (repo_url, install_dir), module_entries in remove_from_mod_json.items(): - for module in module_entries: - self.modules_json["repos"][repo_url]["modules"][install_dir].pop(module) - if len(self.modules_json["repos"][repo_url]["modules"][install_dir]) == 0: - self.modules_json["repos"].pop(repo_url) + resolve_missing_installation(self, missing_installation, "modules") + resolve_missing_installation(self, missing_installation, "subworkflows") - # If some modules didn't have an entry in the 'modules.json' file + # If some modules/subworkflows didn't have an entry in the 'modules.json' file # we try to determine the SHA from the commit log of the remote - if len(missing_from_modules_json) > 0: - format_missing = [f"'{dir}'" for dir in missing_from_modules_json] - if len(format_missing) == 1: - log.info(f"Recomputing commit SHA for module {format_missing[0]} which was missing from 'modules.json'") - else: - log.info( - f"Recomputing commit SHAs for modules which were missing from 'modules.json': {', '.join(format_missing)}" - ) - - # Get the remotes we are missing - tracked_repos = {repo_url: (repo_entry) for repo_url, repo_entry in self.modules_json["repos"].items()} - repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) - - modules_with_repos = ( - ( - nf_core.modules.module_utils.path_from_remote(repo_url), - str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))), - ) - for dir in missing_from_modules_json - for repo_url in repos - if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url)) - ) - - repos_with_modules = {} - for install_dir, module in modules_with_repos: - if install_dir not in repos_with_modules: - repos_with_modules[install_dir] = [] - repos_with_modules[install_dir].append(module) - - for install_dir, modules in repos_with_modules.items(): - remote_url = [url for url, content in repos.items() if install_dir in content][0] - repo_entry = self.determine_module_branches_and_shas(install_dir, remote_url, modules) - if remote_url in self.modules_json["repos"]: - self.modules_json["repos"][remote_url]["modules"][install_dir].update(repo_entry) - else: - self.modules_json["repos"][remote_url] = { - "modules": { - install_dir: repo_entry, - } - } + if len(modules_missing_from_modules_json) > 0: + self.resolve_missing_from_modules_json(modules_missing_from_modules_json, "modules") + if len(subworkflows_missing_from_modules_json) > 0: + self.resolve_missing_from_modules_json(subworkflows_missing_from_modules_json, "subworkflows") self.dump() @@ -909,3 +839,85 @@ def get_installed_subworkflows(self): self.pipeline_subworkflows[repo] = [(dir, name) for name in subworkflow] return self.pipeline_subworkflows + + +def resolve_missing_installation(self, missing_installation, component_type): + missing_but_in_mod_json = [ + f"'{component_type}/{install_dir}/{component}'" + for repo_url, contents in missing_installation.items() + for install_dir, dir_contents in contents[component_type].items() + for component in dir_contents + ] + log.info( + f"Reinstalling {component_type} found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" + ) + + remove_from_mod_json = {} + for repo_url, contents in missing_installation.items(): + for install_dir, component_entries in contents[component_type].items(): + remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( + install_dir, repo_url, component_entries + ) + + # If the reinstall fails, we remove those entries in 'modules.json' + if sum(map(len, remove_from_mod_json.values())) > 0: + uninstallable_components = [ + f"'{install_dir}/{component}'" + for (repo_url, install_dir), components in remove_from_mod_json.items() + for component in components + ] + if len(uninstallable_components) == 1: + log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") + else: + log.info( + f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" + ) + + for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): + for component in component_entries: + self.modules_json["repos"][repo_url][component_type][install_dir].pop(component) + if len(self.modules_json["repos"][repo_url][component_type][install_dir]) == 0: + self.modules_json["repos"].pop(repo_url) + + def resolve_missing_from_modules_json(self, missing_from_modules_json, component_type): + format_missing = [f"'{dir}'" for dir in missing_from_modules_json] + if len(format_missing) == 1: + log.info( + f"Recomputing commit SHA for {component_type[:-1]} {format_missing[0]} which was missing from 'modules.json'" + ) + else: + log.info( + f"Recomputing commit SHAs for {component_type} which were missing from 'modules.json': {', '.join(format_missing)}" + ) + + # Get the remotes we are missing + tracked_repos = {repo_url: (repo_entry) for repo_url, repo_entry in self.modules_json["repos"].items()} + repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) + + components_with_repos = ( + ( + nf_core.modules.module_utils.path_from_remote(repo_url), + str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))), + ) + for dir in missing_from_modules_json + for repo_url in repos + if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url)) + ) + + repos_with_components = {} + for install_dir, component in components_with_repos: + if install_dir not in repos_with_components: + repos_with_components[install_dir] = [] + repos_with_components[install_dir].append(component) + + for install_dir, components in repos_with_components.items(): + remote_url = [url for url, content in repos.items() if install_dir in content][0] + repo_entry = self.determine_module_branches_and_shas(install_dir, remote_url, components) + if remote_url in self.modules_json["repos"]: + self.modules_json["repos"][remote_url][component_type][install_dir].update(repo_entry) + else: + self.modules_json["repos"][remote_url] = { + component_type: { + install_dir: repo_entry, + } + } From 098518af49547c5f8f5ae394509306e1adc94fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jennifer=20M=C3=BCller?= <43847598+jenmuell@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:20:05 +0200 Subject: [PATCH 255/854] Added missing dot Added a missing dot in the `Launch a Pipeline` section of Tools https://github.com/nf-core/tools/issues/1932 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ad2846b23..0e0c518479 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ Archived pipelines are not returned by default. To include them, use the `--show ## Launch a pipeline Some nextflow pipelines have a considerable number of command line flags that can be used. -To help with this, you can use the `nf-core launch` command +To help with this, you can use the `nf-core launch` command. You can choose between a web-based graphical interface or an interactive command-line wizard tool to enter the pipeline parameters for your run. Both interfaces show documentation alongside each parameter and validate your inputs. From 8b1d8f861546081f6c4f3113f93f135e4cd2f644 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 11 Oct 2022 16:30:05 +0200 Subject: [PATCH 256/854] fix silly bugs :) --- nf_core/modules/modules_json.py | 69 ++++++++++++++++----------------- nf_core/subworkflows/install.py | 10 ++--- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 26cb755a43..c26f835731 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -840,44 +840,43 @@ def get_installed_subworkflows(self): return self.pipeline_subworkflows + def resolve_missing_installation(self, missing_installation, component_type): + missing_but_in_mod_json = [ + f"'{component_type}/{install_dir}/{component}'" + for repo_url, contents in missing_installation.items() + for install_dir, dir_contents in contents[component_type].items() + for component in dir_contents + ] + log.info( + f"Reinstalling {component_type} found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" + ) -def resolve_missing_installation(self, missing_installation, component_type): - missing_but_in_mod_json = [ - f"'{component_type}/{install_dir}/{component}'" - for repo_url, contents in missing_installation.items() - for install_dir, dir_contents in contents[component_type].items() - for component in dir_contents - ] - log.info( - f"Reinstalling {component_type} found in 'modules.json' but missing from directory: {', '.join(missing_but_in_mod_json)}" - ) - - remove_from_mod_json = {} - for repo_url, contents in missing_installation.items(): - for install_dir, component_entries in contents[component_type].items(): - remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( - install_dir, repo_url, component_entries - ) + remove_from_mod_json = {} + for repo_url, contents in missing_installation.items(): + for install_dir, component_entries in contents[component_type].items(): + remove_from_mod_json[(repo_url, install_dir)] = self.reinstall_repo( + install_dir, repo_url, component_entries + ) - # If the reinstall fails, we remove those entries in 'modules.json' - if sum(map(len, remove_from_mod_json.values())) > 0: - uninstallable_components = [ - f"'{install_dir}/{component}'" - for (repo_url, install_dir), components in remove_from_mod_json.items() - for component in components - ] - if len(uninstallable_components) == 1: - log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") - else: - log.info( - f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" - ) + # If the reinstall fails, we remove those entries in 'modules.json' + if sum(map(len, remove_from_mod_json.values())) > 0: + uninstallable_components = [ + f"'{install_dir}/{component}'" + for (repo_url, install_dir), components in remove_from_mod_json.items() + for component in components + ] + if len(uninstallable_components) == 1: + log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") + else: + log.info( + f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" + ) - for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): - for component in component_entries: - self.modules_json["repos"][repo_url][component_type][install_dir].pop(component) - if len(self.modules_json["repos"][repo_url][component_type][install_dir]) == 0: - self.modules_json["repos"].pop(repo_url) + for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): + for component in component_entries: + self.modules_json["repos"][repo_url][component_type][install_dir].pop(component) + if len(self.modules_json["repos"][repo_url][component_type][install_dir]) == 0: + self.modules_json["repos"].pop(repo_url) def resolve_missing_from_modules_json(self, missing_from_modules_json, component_type): format_missing = [f"'{dir}'" for dir in missing_from_modules_json] diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index de286e0e72..f17067fa2e 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -53,11 +53,11 @@ def install(self, subworkflow, silent=False): # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - if "subworkflows" not in modules_json.modules_json["repos"][self.modules_repo.remote_url]: - # It's the first subworkflow installed in the pipeline! - modules_json.modules_json["repos"][self.modules_repo.remote_url]["subworkflows"] = { - self.modules_repo.repo_path: {} - } + # if "subworkflows" not in modules_json.modules_json["repos"][self.modules_repo.remote_url]: + # # It's the first subworkflow installed in the pipeline! + # modules_json.modules_json["repos"][self.modules_repo.remote_url]["subworkflows"] = { + # self.modules_repo.repo_path: {} + # } if self.prompt and self.sha is not None: log.error("Cannot use '--sha' and '--prompt' at the same time!") From 9c517b4aff4b049a68b38374a291e6e3b928fd8e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 11 Oct 2022 16:30:58 +0200 Subject: [PATCH 257/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 272b5398a9..e5d0479b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Don't remove local copy of modules repo, only update it with fetch ([#1881](https://github.com/nf-core/tools/pull/1881)) - Add subworkflow commands create-test-yml, create and install ([#1897](https://github.com/nf-core/tools/pull/1897)) - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) +- `check_up_to_date()` function from `modules_json` also checks for subworkflows. ### Modules From 1d2ee9b38cb3c21cb2844af217ee4fcc75f23b80 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 10:34:03 +0200 Subject: [PATCH 258/854] Test reset_target_dir --- nf_core/sync.py | 2 +- tests/test_sync.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index a663f1c7a7..7fdbb3d2c0 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -423,4 +423,4 @@ def reset_target_dir(self): try: self.repo.git.checkout(self.original_branch) except GitCommandError as e: - raise SyncException(f"Could not reset to original branch `{self.from_branch}`:\n{e}") + raise SyncException(f"Could not reset to original branch `{self.original_branch}`:\n{e}") diff --git a/tests/test_sync.py b/tests/test_sync.py index 2779f9e356..9c3e241db4 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -4,6 +4,7 @@ import json import os +import pytest import shutil import tempfile import unittest @@ -251,3 +252,27 @@ def test_make_pull_request_bad_response(self, mock_post, mock_get): raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("Something went badly wrong - GitHub API PR failed - got return code 404") + + def test_reset_target_dir(self): + """Try resetting target pipeline directory""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + + psync.repo.git.checkout("dev") + + psync.reset_target_dir() + + assert psync.repo.heads[0].name == "TEMPLATE" + + def test_reset_target_dir_fake_branch(self): + """Try resetting target pipeline directory but original branch does not exist""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + + psync.original_branch = "fake_branch" + + with pytest.raises(nf_core.sync.SyncException) as exc_info: + psync.reset_target_dir() + assert exc_info.value.args[0].startswith("Could not reset to original branch `fake_branch`") From 9855cf20b2818ce16b002d35808c651a9105c02d Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 10:35:00 +0200 Subject: [PATCH 259/854] Test checkout_template_branch raises exception --- tests/test_sync.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_sync.py b/tests/test_sync.py index 9c3e241db4..7e856cb6f0 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -92,6 +92,18 @@ def test_checkout_template_branch(self): psync.get_wf_config() psync.checkout_template_branch() + def test_checkout_template_branch_no_template(self): + """Try checking out the TEMPLATE branch of the pipeline when it does not exist""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + + psync.repo.delete_head("TEMPLATE") + + with pytest.raises(nf_core.sync.SyncException) as exc_info: + psync.checkout_template_branch() + assert exc_info.value.args[0] == "Could not check out branch 'origin/TEMPLATE' or 'TEMPLATE'" + def test_delete_template_branch_files(self): """Confirm that we can delete all files in the TEMPLATE branch""" psync = nf_core.sync.PipelineSync(self.pipeline_dir) From ceb30d9a222d350a6b7137b63b2992cd9b27f1b6 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 11:01:37 +0200 Subject: [PATCH 260/854] Test create_merge_base branch There was a bug in this function: because `self.merge_branch` was used to store the new branch name, it was stored persistently in the `psync` object. Hence when a new branch was created, the branch number was appended several times, e.g. `original_merge_branch-2-3-4`. --- nf_core/sync.py | 25 ++++++++++++++----------- tests/test_sync.py | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 7fdbb3d2c0..4f9809a59e 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -8,6 +8,7 @@ import shutil import git +import re import requests import requests_cache import rich @@ -67,7 +68,8 @@ def __init__( self.pipeline_dir = os.path.abspath(pipeline_dir) self.from_branch = from_branch self.original_branch = None - self.merge_branch = f"nf-core-template-merge-{nf_core.__version__}" + self.original_merge_branch = f"nf-core-template-merge-{nf_core.__version__}" + self.merge_branch = self.original_merge_branch self.made_changes = False self.make_pr = make_pr self.gh_pr_returned_data = {} @@ -270,17 +272,18 @@ def create_merge_base_branch(self): # Check if branch exists already branch_list = [b.name for b in self.repo.branches] if self.merge_branch in branch_list: - original_merge_branch = self.merge_branch - # Try to create new branch with number at the end - # If -2 already exists, increase the number until branch is new - branch_no = 2 - self.merge_branch = f"{original_merge_branch}-{branch_no}" - while self.merge_branch in branch_list: - branch_no += 1 - self.merge_branch = f"{original_merge_branch}-{branch_no}" - log.info( - f"Branch already existed: '{original_merge_branch}', creating branch '{self.merge_branch}' instead." + merge_branch_format = re.compile(rf"{self.original_merge_branch}-(\d+)") + max_branch = max( + [1] + + [ + int(merge_branch_format.match(branch).groups()[0]) + for branch in branch_list + if merge_branch_format.match(branch) + ] ) + new_branch = f"{self.original_merge_branch}-{max_branch+1}" + log.info(f"Branch already existed: '{self.merge_branch}', creating branch '{new_branch}' instead.") + self.merge_branch = new_branch # Create new branch and checkout log.info(f"Checking out merge base branch '{self.merge_branch}'") diff --git a/tests/test_sync.py b/tests/test_sync.py index 7e856cb6f0..afd1576809 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -172,6 +172,29 @@ def test_push_template_branch_error(self): except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("Could not push TEMPLATE branch") + def test_create_merge_base_branch(self): + """Try creating a merge base branch""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + + psync.create_merge_base_branch() + + assert psync.merge_branch in psync.repo.branches + + def test_create_merge_base_branch_thrice(self): + """Try creating a merge base branch thrice""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + + for _ in range(3): + psync.create_merge_base_branch() + + assert psync.merge_branch in psync.repo.branches + for branch_no in [2, 3]: + assert f"{psync.original_merge_branch}-{branch_no}" in psync.repo.branches + def mocked_requests_get(url, **kwargs): """Helper function to emulate POST requests responses from the web""" From 373f596524906df4fa9105d83b3ff0c8f12c8b66 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 14:16:15 +0200 Subject: [PATCH 261/854] Test push_merge_branch --- tests/test_sync.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_sync.py b/tests/test_sync.py index afd1576809..8291840239 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -2,6 +2,7 @@ """ Tests covering the sync command """ +import git import json import os import pytest @@ -27,6 +28,8 @@ def setUp(self): "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True ) self.create_obj.init_pipeline() + self.remote_path = os.path.join(self.tmp_dir, "remote_repo") + self.remote_repo = git.Repo.init(self.remote_path, bare=True) def tearDown(self): if os.path.exists(self.tmp_dir): @@ -195,6 +198,29 @@ def test_create_merge_base_branch_thrice(self): for branch_no in [2, 3]: assert f"{psync.original_merge_branch}-{branch_no}" in psync.repo.branches + def test_push_merge_branch(self): + """Try pushing merge branch""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + psync.repo.create_remote("origin", self.remote_path) + + psync.create_merge_base_branch() + psync.push_merge_branch() + + assert psync.merge_branch in [b.name for b in self.remote_repo.branches] + + def test_push_merge_branch_without_create_branch(self): + """Try pushing merge branch without creating first""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + psync.repo.create_remote("origin", self.remote_path) + + with pytest.raises(nf_core.sync.PullRequestException) as exc_info: + psync.push_merge_branch() + assert exc_info.value.args[0].startswith(f"Could not push branch '{psync.merge_branch}'") + def mocked_requests_get(url, **kwargs): """Helper function to emulate POST requests responses from the web""" From a8c99b0ad5e5f179d0c18cda8e3ea581eaee7317 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 15:43:49 +0200 Subject: [PATCH 262/854] Test close_open_template_merge_prs --- tests/test_sync.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 8291840239..1450f78e79 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -237,10 +237,28 @@ def __init__(self, data, status_code): def json(self): return self.data - url_template = "https://api.github.com/repos/{}/response/pulls?head=TEMPLATE&base=None" - if url == url_template.format("no_existing_pr"): + url_template = "https://api.github.com/repos/{}/response/" + if url == os.path.join(url_template.format("no_existing_pr"), "pulls?head=TEMPLATE&base=None"): response_data = [] return MockResponse(response_data, 200) + elif url == os.path.join(url_template.format("list_prs"), "pulls"): + response_data = [ + { + "state": "closed", + "head": {"ref": "nf-core-template-merge-2"}, + "base": {"ref": "master"}, + "html_url": "pr_url", + } + ] + [ + { + "state": "open", + "head": {"ref": f"nf-core-template-merge-{branch_no}"}, + "base": {"ref": "master"}, + "html_url": "pr_url", + } + for branch_no in range(3, 7) + ] + return MockResponse(response_data, 200) return MockResponse({"html_url": url}, 404) @@ -314,6 +332,27 @@ def test_make_pull_request_bad_response(self, mock_post, mock_get): except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("Something went badly wrong - GitHub API PR failed - got return code 404") + @mock.patch("nf_core.utils.gh_api.get", side_effect=mocked_requests_get) + def test_close_open_template_merge_prs(self, mock_get): + """Try closing all open prs""" + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + psync.gh_api.get = mock_get + psync.gh_username = "list_prs" + psync.gh_repo = "list_prs/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" + + with mock.patch("nf_core.sync.PipelineSync.close_open_pr") as mock_close_open_pr: + psync.close_open_template_merge_prs() + + prs = mock_get(f"https://api.github.com/repos/{psync.gh_repo}/pulls").data + for pr in prs: + if pr["state"] != "open": + continue + else: + mock_close_open_pr.assert_any_call(pr) + def test_reset_target_dir(self): """Try resetting target pipeline directory""" psync = nf_core.sync.PipelineSync(self.pipeline_dir) From 005fecf6d2c6b4e82be5ee0c9076e0926aaf0591 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 16:34:12 +0200 Subject: [PATCH 263/854] Test close_open_pr --- tests/test_sync.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_sync.py b/tests/test_sync.py index 1450f78e79..312675ab30 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -353,6 +353,52 @@ def test_close_open_template_merge_prs(self, mock_get): else: mock_close_open_pr.assert_any_call(pr) + @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) + @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) + def test_close_open_pr(self, mock_patch, mock_post): + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + psync.gh_api.post = mock_post + psync.gh_api.patch = mock_patch + psync.gh_username = "bad_url" + psync.gh_repo = "bad_url/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" + pr = { + "state": "open", + "head": {"ref": "nf-core-template-merge-3"}, + "base": {"ref": "master"}, + "html_url": "pr_html_url", + "url": "url_to_update_pr", + "comments_url": "pr_comments_url", + } + + assert psync.close_open_pr(pr) + assert mock_patch.called_once_with("url_to_update_pr") + + @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) + @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) + def test_close_open_pr_fail(self, mock_patch, mock_post): + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.inspect_sync_dir() + psync.get_wf_config() + psync.gh_api.post = mock_post + psync.gh_api.patch = mock_patch + psync.gh_username = "bad_url" + psync.gh_repo = "bad_url/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" + pr = { + "state": "open", + "head": {"ref": "nf-core-template-merge-3"}, + "base": {"ref": "master"}, + "html_url": "pr_html_url", + "url": "bad_url_to_update_pr", + "comments_url": "pr_comments_url", + } + + assert not psync.close_open_pr(pr) + assert mock_patch.called_once_with("bad_url_to_update_pr") + def test_reset_target_dir(self): """Try resetting target pipeline directory""" psync = nf_core.sync.PipelineSync(self.pipeline_dir) From bb4b621b03ef4c1d4244dd1fedddf82e70b1333e Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 16:52:00 +0200 Subject: [PATCH 264/854] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 272b5398a9..aa90d41e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Don't remove local copy of modules repo, only update it with fetch ([#1881](https://github.com/nf-core/tools/pull/1881)) - Add subworkflow commands create-test-yml, create and install ([#1897](https://github.com/nf-core/tools/pull/1897)) - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) +- Improve test coverage of sync.py ### Modules From 8fbbef4dbd9aad30713551c54c15022b2d759cb2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 11 Oct 2022 16:53:04 +0200 Subject: [PATCH 265/854] fix linting in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ce16f65a9c..ce1e27f9ce 100644 --- a/README.md +++ b/README.md @@ -190,11 +190,13 @@ export NFCORE_NO_VERSION_CHECK=1 ### Update tools It is advisable to keep nf-core/tools updated to the most recent version. The command to update depends on the system used to install it, for example if you have installed it with conda you can use: + ```bash conda update nf-core ``` if you used pip: + ```bash pip install --upgrade nf-core ``` From 7c3225880f5c85f2b0b2282c9df82c37d16339a2 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 17:00:33 +0200 Subject: [PATCH 266/854] Convert exception checks to pytest.raises --- tests/test_sync.py | 49 ++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 312675ab30..9bd899119f 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -39,11 +39,9 @@ def tearDown(self): def test_inspect_sync_dir_notgit(self, tmp_dir): """Try syncing an empty directory""" psync = nf_core.sync.PipelineSync(tmp_dir) - try: + with pytest.raises(nf_core.sync.SyncException) as exc_info: psync.inspect_sync_dir() - raise UserWarning("Should have hit an exception") - except nf_core.sync.SyncException as e: - assert "does not appear to be a git repository" in e.args[0] + assert "does not appear to be a git repository" in exc_info.value.args[0] def test_inspect_sync_dir_dirty(self): """Try syncing a pipeline with uncommitted changes""" @@ -53,40 +51,33 @@ def test_inspect_sync_dir_dirty(self): # Try to sync, check we halt with the right error psync = nf_core.sync.PipelineSync(self.pipeline_dir) try: - psync.inspect_sync_dir() - raise UserWarning("Should have hit an exception") - except nf_core.sync.SyncException as e: - os.remove(test_fn) - assert e.args[0].startswith("Uncommitted changes found in pipeline directory!") - except Exception as e: + with pytest.raises(nf_core.sync.SyncException) as exc_info: + psync.inspect_sync_dir() + assert exc_info.value.args[0].startswith("Uncommitted changes found in pipeline directory!") + finally: os.remove(test_fn) - raise e def test_get_wf_config_no_branch(self): """Try getting a workflow config when the branch doesn't exist""" # Try to sync, check we halt with the right error psync = nf_core.sync.PipelineSync(self.pipeline_dir, from_branch="foo") - try: + with pytest.raises(nf_core.sync.SyncException) as exc_info: psync.inspect_sync_dir() psync.get_wf_config() - raise UserWarning("Should have hit an exception") - except nf_core.sync.SyncException as e: - assert e.args[0] == "Branch `foo` not found!" + assert exc_info.value.args[0] == "Branch `foo` not found!" def test_get_wf_config_missing_required_config(self): """Try getting a workflow config, then make it miss a required config option""" # Try to sync, check we halt with the right error psync = nf_core.sync.PipelineSync(self.pipeline_dir) psync.required_config_vars = ["fakethisdoesnotexist"] - try: + with pytest.raises(nf_core.sync.SyncException) as exc_info: psync.inspect_sync_dir() psync.get_wf_config() - raise UserWarning("Should have hit an exception") - except nf_core.sync.SyncException as e: - # Check that we did actually get some config back - assert psync.wf_config["params.validate_params"] == "true" - # Check that we raised because of the missing fake config var - assert e.args[0] == "Workflow config variable `fakethisdoesnotexist` not found!" + # Check that we did actually get some config back + assert psync.wf_config["params.validate_params"] == "true" + # Check that we raised because of the missing fake config var + assert exc_info.value.args[0] == "Workflow config variable `fakethisdoesnotexist` not found!" def test_checkout_template_branch(self): """Try checking out the TEMPLATE branch of the pipeline""" @@ -169,11 +160,9 @@ def test_push_template_branch_error(self): open(test_fn, "a").close() psync.commit_template_changes() # Try to push changes - try: + with pytest.raises(nf_core.sync.PullRequestException) as exc_info: psync.push_template_branch() - raise UserWarning("Should have hit an exception") - except nf_core.sync.PullRequestException as e: - assert e.args[0].startswith("Could not push TEMPLATE branch") + assert exc_info.value.args[0].startswith("Could not push TEMPLATE branch") def test_create_merge_base_branch(self): """Try creating a merge base branch""" @@ -326,11 +315,11 @@ def test_make_pull_request_bad_response(self, mock_post, mock_get): psync.gh_username = "bad_url" psync.gh_repo = "bad_url/response" os.environ["GITHUB_AUTH_TOKEN"] = "test" - try: + with pytest.raises(nf_core.sync.PullRequestException) as exc_info: psync.make_pull_request() - raise UserWarning("Should have hit an exception") - except nf_core.sync.PullRequestException as e: - assert e.args[0].startswith("Something went badly wrong - GitHub API PR failed - got return code 404") + assert exc_info.value.args[0].startswith( + "Something went badly wrong - GitHub API PR failed - got return code 404" + ) @mock.patch("nf_core.utils.gh_api.get", side_effect=mocked_requests_get) def test_close_open_template_merge_prs(self, mock_get): From 538cb8d1b91fa57d1cb08515ebf40679cc28af38 Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Tue, 11 Oct 2022 17:08:49 +0200 Subject: [PATCH 267/854] add tests for "subworkflow list" --- tests/subworkflows/list.py | 48 ++++++++++++++++++++++++++++++++++++++ tests/test_subworkflows.py | 18 ++++++++++---- 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 tests/subworkflows/list.py diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py new file mode 100644 index 0000000000..d6f58db63f --- /dev/null +++ b/tests/subworkflows/list.py @@ -0,0 +1,48 @@ +from rich.console import Console + +import nf_core.modules + +from ..utils import GITLAB_DEFAULT_BRANCH, GITLAB_URL + + +def test_subworkflows_list_remote(self): + """Test listing available subworkflows""" + swfs_list = nf_core.subworkflows.SubworkflowList(None, remote=True) + listed_swfs = swfs_list.list_subworkflows() + console = Console(record=True) + console.print(listed_swfs) + output = console.export_text() + assert "bam_sort_samtools" in output + + +# def test_subworkflows_list_remote_gitlab(self): +# """Test listing the subworkflows in the remote gitlab repo""" +# swfs_list = nf_core.subworkflows.SubworkflowList(None, remote=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH) +# listed_swfs = swfs_list.list_subworkflows() +# print(f"listed subworkflows are {listed_swfs}") +# console = Console(record=True) +# console.print(listed_swfs) +# output = console.export_text() +# assert "bam_sort_samtools" in output + + +def test_subworkflows_install_and_list_pipeline(self): + """Test listing locally installed subworkflows""" + self.swfs_install.install("align_bowtie2") + swfs_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) + listed_swfs = swfs_list.list_subworkflows() + console = Console(record=True) + console.print(listed_swfs) + output = console.export_text() + assert "align_bowtie2" in output + + +# def test_subworkflows_install_gitlab_and_list_pipeline(self): +# """Test listing locally installed subworkflows""" +# self.swfs_install_gitlab.install("bam_sort_samtools") +# swfs_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) +# listed_swfs = swfs_list.list_subworkflows() +# console = Console(record=True) +# console.print(listed_swfs) +# output = console.export_text() +# assert "bam_sort_samtools" in output diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index e5d3f7d3f4..7eb491d809 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -51,6 +51,9 @@ def setUp(self): "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() + # Set up install objects + self.swfs_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=True) + # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) @@ -58,8 +61,15 @@ def setUp(self): # Test of the individual modules commands. # ############################################ - from .subworkflows.create import ( - test_subworkflows_create_fail_exists, - test_subworkflows_create_nfcore_modules, - test_subworkflows_create_succeed, + # from .subworkflows.create import ( + # test_subworkflows_create_fail_exists, + # test_subworkflows_create_nfcore_modules, + # test_subworkflows_create_succeed, + # ) + + from .subworkflows.list import ( + # test_subworkflows_install_gitlab_and_list_pipeline, + test_subworkflows_install_and_list_pipeline, + test_subworkflows_list_remote, + # test_subworkflows_list_remote_gitlab, ) From 3fa8ed1395fed4e807e5991708083e3adad98740 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 11 Oct 2022 17:12:39 +0200 Subject: [PATCH 268/854] Sort imports --- nf_core/sync.py | 2 +- tests/test_sync.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 4f9809a59e..c4351d96fe 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -5,10 +5,10 @@ import json import logging import os +import re import shutil import git -import re import requests import requests_cache import rich diff --git a/tests/test_sync.py b/tests/test_sync.py index 9bd899119f..9d6eb5f488 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -2,15 +2,16 @@ """ Tests covering the sync command """ -import git import json import os -import pytest import shutil import tempfile import unittest from unittest import mock +import git +import pytest + import nf_core.create import nf_core.sync From d211ac557e872a15dc3510e5e5efc757746b9a2c Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Tue, 11 Oct 2022 17:14:14 +0200 Subject: [PATCH 269/854] fix python-isort and un-comment test --- tests/test_subworkflows.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 7eb491d809..5b947c5e8f 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -61,15 +61,13 @@ def setUp(self): # Test of the individual modules commands. # ############################################ - # from .subworkflows.create import ( - # test_subworkflows_create_fail_exists, - # test_subworkflows_create_nfcore_modules, - # test_subworkflows_create_succeed, - # ) + from .subworkflows.create import ( + test_subworkflows_create_fail_exists, + test_subworkflows_create_nfcore_modules, + test_subworkflows_create_succeed, + ) from .subworkflows.list import ( - # test_subworkflows_install_gitlab_and_list_pipeline, test_subworkflows_install_and_list_pipeline, test_subworkflows_list_remote, - # test_subworkflows_list_remote_gitlab, ) From 68583719044640391e0fccb60f73d7278954c006 Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Tue, 11 Oct 2022 17:29:37 +0200 Subject: [PATCH 270/854] still fixing python-isort --- tests/test_subworkflows.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 5b947c5e8f..dd95c6b02a 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -66,7 +66,6 @@ def setUp(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) - from .subworkflows.list import ( test_subworkflows_install_and_list_pipeline, test_subworkflows_list_remote, From fa5319ae24aec2727a060a38d7cf1d840f6ba04a Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 11 Oct 2022 17:50:11 +0200 Subject: [PATCH 271/854] Remove unused imports --- nf_core/modules/modules_command.py | 8 -------- nf_core/subworkflows/install.py | 1 - 2 files changed, 9 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 8df2ee2232..4f0a6f26c3 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -1,16 +1,8 @@ import logging -import os -import shutil from pathlib import Path -import yaml from nf_core.components.components import Components -import nf_core.modules.module_utils -import nf_core.utils - -from .modules_json import ModulesJson -from .modules_repo import ModulesRepo log = logging.getLogger(__name__) diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 9d4363f3ff..ebfca6f147 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -1,7 +1,6 @@ import logging import os import re -import shutil from pathlib import Path import questionary From dd6f5d1a680bcab88fe7319c54d9037fc757bf65 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 11 Oct 2022 17:51:03 +0200 Subject: [PATCH 272/854] Implement SubworkflowCommand and use it as base for SubworkflowInstall --- nf_core/subworkflows/install.py | 53 +------------- nf_core/subworkflows/subworkflow_command.py | 81 +++++++++++++++++++++ 2 files changed, 85 insertions(+), 49 deletions(-) create mode 100644 nf_core/subworkflows/subworkflow_command.py diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index ebfca6f147..69e7983d9d 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -9,14 +9,14 @@ import nf_core.utils from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME -# from .modules_command import ModuleCommand -from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, ModulesRepo +from .subworkflow_command import SubworkflowCommand log = logging.getLogger(__name__) -class SubworkflowInstall(object): +class SubworkflowInstall(SubworkflowCommand): def __init__( self, pipeline_dir, @@ -27,19 +27,10 @@ def __init__( branch=None, no_pull=False, ): - # super().__init__(pipeline_dir, remote_url, branch, no_pull) - self.dir = pipeline_dir + super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha - self.modules_repo = ModulesRepo(remote_url, branch, no_pull) - try: - if self.dir: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) - else: - self.repo_type = None - except LookupError as e: - raise UserWarning(e) def install(self, subworkflow, silent=False): if self.repo_type == "modules": @@ -181,42 +172,6 @@ def install(self, subworkflow, silent=False): modules_json.update_subworkflow(self.modules_repo, subworkflow, version) return True - def has_valid_directory(self): - """Check that we were given a pipeline""" - if self.dir is None or not os.path.exists(self.dir): - log.error(f"Could not find pipeline: {self.dir}") - return False - main_nf = os.path.join(self.dir, "main.nf") - nf_config = os.path.join(self.dir, "nextflow.config") - if not os.path.exists(main_nf) and not os.path.exists(nf_config): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") - return True - - def clear_subworkflow_dir(self, subworkflow_name, subworkflow_dir): - """Removes all files in the subworkflow directory""" - try: - shutil.rmtree(subworkflow_dir) - log.debug(f"Successfully removed {subworkflow_name} subworkflow") - return True - except OSError as e: - log.error(f"Could not remove subworkflow: {e}") - return False - - def install_subworkflow_files(self, subworkflow_name, subworkflow_version, modules_repo, install_dir): - """ - Installs a subworkflow into the given directory - - Args: - subworkflow_name (str): The name of the subworkflow - subworkflow_version (str): Git SHA for the version of the subworkflow to be installed - modules_repo (ModulesRepo): A correctly configured ModulesRepo object - install_dir (str): The path to where the subworkflow should be installed (should be the 'subworkflow/' dir of the pipeline) - - Returns: - (bool): Whether the operation was successful of not - """ - return modules_repo.install_subworkflow(subworkflow_name, install_dir, subworkflow_version) - def get_modules_subworkflows_to_install(self, subworkflow_dir): """ Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows. diff --git a/nf_core/subworkflows/subworkflow_command.py b/nf_core/subworkflows/subworkflow_command.py new file mode 100644 index 0000000000..38fa58a497 --- /dev/null +++ b/nf_core/subworkflows/subworkflow_command.py @@ -0,0 +1,81 @@ +import logging +import os +import shutil +from pathlib import Path + +from nf_core.components.components import Components + + +log = logging.getLogger(__name__) + + +class SubworkflowCommand(Components): + """ + Base class for the 'nf-core subworkflows' commands + """ + + def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): + self.default_subworkflows_path = Path("subworkflows", "nf-core") + self.default_subworkflows_tests_path = Path("tests", "subworkflows", "nf-core") + super().__init__("subworkflows", pipeline_dir, remote_url, branch, no_pull) + + def get_subworkflows_clone_subworkflows(self): + """ + Get the subworkflows repository available in a clone of nf-core/subworkflows + """ + subworkflow_base_path = Path(self.dir, self.default_subworkflows_path) + return [ + str(Path(dir).relative_to(subworkflow_base_path)) + for dir, _, files in os.walk(subworkflow_base_path) + if "main.nf" in files + ] + + def get_local_subworkflows(self): + """ + Get the local subworkflows in a pipeline + """ + local_subwf_dir = Path(self.dir, "subworkflows", "local") + return [str(path.relative_to(local_subwf_dir)) for path in local_subwf_dir.iterdir() if path.suffix == ".nf"] + + def subworkflows_from_repo(self, install_dir): + """ + Gets the subworkflows installed from a certain repository + + Args: + install_dir (str): The name of the directory where subworkflows are installed + + Returns: + [str]: The names of the subworkflows + """ + repo_dir = Path(self.dir, "subworkflows", install_dir) + if not repo_dir.exists(): + raise LookupError(f"Nothing installed from {install_dir} in pipeline") + + return [ + str(Path(dir_path).relative_to(repo_dir)) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files + ] + + def install_subworkflow_files(self, subworkflow_name, subworkflow_version, modules_repo, install_dir): + """ + Installs a subworkflow into the given directory + + Args: + subworkflow_name (str): The name of the subworkflow + subworkflow_version (str): Git SHA for the version of the subworkflow to be installed + modules_repo (ModulesRepo): A correctly configured ModulesRepo object + install_dir (str): The path to where the subworkflow should be installed (should be the 'subworkflow/' dir of the pipeline) + + Returns: + (bool): Whether the operation was successful of not + """ + return modules_repo.install_subworkflow(subworkflow_name, install_dir, subworkflow_version) + + def clear_subworkflow_dir(self, subworkflow_name, subworkflow_dir): + """Removes all files in the subworkflow directory""" + try: + shutil.rmtree(subworkflow_dir) + log.debug(f"Successfully removed {subworkflow_name} subworkflow") + return True + except OSError as e: + log.error(f"Could not remove subworkflow: {e}") + return False From e4cd37fe3567ce6860cfc073fcd9f372916be019 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:02:59 +0200 Subject: [PATCH 273/854] Add __init__.py to components so it's recognised as a module --- nf_core/components/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 nf_core/components/__init__.py diff --git a/nf_core/components/__init__.py b/nf_core/components/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 1033a5686b2e203ad3f36ab862272e154f7502de Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:11:21 +0200 Subject: [PATCH 274/854] Change pipeline_dir to dir as arg name to be consistent with old ModuleCommand class --- nf_core/modules/modules_command.py | 4 ++-- nf_core/subworkflows/subworkflow_command.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 4f0a6f26c3..de6b51249b 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -12,8 +12,8 @@ class ModuleCommand(Components): Base class for the 'nf-core modules' commands """ - def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): - super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) + def __init__(self, dir, remote_url=None, branch=None, no_pull=False): + super().__init__("modules", dir, remote_url, branch, no_pull) def get_local_modules(self): """ diff --git a/nf_core/subworkflows/subworkflow_command.py b/nf_core/subworkflows/subworkflow_command.py index 38fa58a497..563697f6b0 100644 --- a/nf_core/subworkflows/subworkflow_command.py +++ b/nf_core/subworkflows/subworkflow_command.py @@ -14,10 +14,10 @@ class SubworkflowCommand(Components): Base class for the 'nf-core subworkflows' commands """ - def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): + def __init__(self, dir, remote_url=None, branch=None, no_pull=False): self.default_subworkflows_path = Path("subworkflows", "nf-core") self.default_subworkflows_tests_path = Path("tests", "subworkflows", "nf-core") - super().__init__("subworkflows", pipeline_dir, remote_url, branch, no_pull) + super().__init__("subworkflows", dir, remote_url, branch, no_pull) def get_subworkflows_clone_subworkflows(self): """ From 4247a792bd6f5b67d68919b203ef5c8cb202af55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 12 Oct 2022 10:29:20 +0200 Subject: [PATCH 275/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/modules/modules_json.py | 8 ++++---- nf_core/subworkflows/install.py | 5 ----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index c26f835731..e538f3ef27 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -364,7 +364,7 @@ def unsynced_modules(self): def parse_dirs(self, dirs, missing_installation, component_type): untracked_dirs = [] for dir in dirs: - # Check if the module/ssubworkflows directory exists in modules.json + # Check if the module/subworkflows directory exists in modules.json install_dir = dir.parts[0] component = str(Path(*dir.parts[1:])) component_in_file = False @@ -388,7 +388,7 @@ def parse_dirs(self, dirs, missing_installation, component_type): # Remove the subworkflow from subworkflows without installation module_repo[component_type][install_dir].pop(component) if len(module_repo[component_type][install_dir]) == 0: - # If no subworkflows with missing installation left, remove the git_url from missing_installation + # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation missing_installation.pop(git_url) return untracked_dirs, missing_installation @@ -482,8 +482,8 @@ def check_up_to_date(self): # If there are any modules/subworkflows left in 'modules.json' after all installed are removed, # we try to reinstall them if len(missing_installation) > 0: - resolve_missing_installation(self, missing_installation, "modules") resolve_missing_installation(self, missing_installation, "subworkflows") + resolve_missing_installation(self, missing_installation, "modules") # If some modules/subworkflows didn't have an entry in the 'modules.json' file # we try to determine the SHA from the commit log of the remote @@ -869,7 +869,7 @@ def resolve_missing_installation(self, missing_installation, component_type): log.info(f"Was unable to reinstall {uninstallable_components[0]}. Removing 'modules.json' entry") else: log.info( - f"Was unable to reinstall some modules. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" + f"Was unable to reinstall some {component_type}. Removing 'modules.json' entries: {', '.join(uninstallable_components)}" ) for (repo_url, install_dir), component_entries in remove_from_mod_json.items(): diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index f17067fa2e..f7e495b53b 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -53,11 +53,6 @@ def install(self, subworkflow, silent=False): # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - # if "subworkflows" not in modules_json.modules_json["repos"][self.modules_repo.remote_url]: - # # It's the first subworkflow installed in the pipeline! - # modules_json.modules_json["repos"][self.modules_repo.remote_url]["subworkflows"] = { - # self.modules_repo.repo_path: {} - # } if self.prompt and self.sha is not None: log.error("Cannot use '--sha' and '--prompt' at the same time!") From c324a57d20bf27dcbe0a821c3d41b2156be3d52e Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:31:21 +0200 Subject: [PATCH 276/854] Rename subworkflows_command module to be consistent with the modules one. Pass-through for hide_progress arg --- nf_core/modules/modules_command.py | 4 ++-- .../{subworkflow_command.py => subworkflows_command.py} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename nf_core/subworkflows/{subworkflow_command.py => subworkflows_command.py} (98%) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index de6b51249b..3bd176ef11 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -12,8 +12,8 @@ class ModuleCommand(Components): Base class for the 'nf-core modules' commands """ - def __init__(self, dir, remote_url=None, branch=None, no_pull=False): - super().__init__("modules", dir, remote_url, branch, no_pull) + def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): + super().__init__("modules", dir, remote_url, branch, no_pull, hide_progress) def get_local_modules(self): """ diff --git a/nf_core/subworkflows/subworkflow_command.py b/nf_core/subworkflows/subworkflows_command.py similarity index 98% rename from nf_core/subworkflows/subworkflow_command.py rename to nf_core/subworkflows/subworkflows_command.py index 563697f6b0..65de789057 100644 --- a/nf_core/subworkflows/subworkflow_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -14,10 +14,10 @@ class SubworkflowCommand(Components): Base class for the 'nf-core subworkflows' commands """ - def __init__(self, dir, remote_url=None, branch=None, no_pull=False): + def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): self.default_subworkflows_path = Path("subworkflows", "nf-core") self.default_subworkflows_tests_path = Path("tests", "subworkflows", "nf-core") - super().__init__("subworkflows", dir, remote_url, branch, no_pull) + super().__init__("subworkflows", dir, remote_url, branch, no_pull, hide_progress) def get_subworkflows_clone_subworkflows(self): """ From e123f57d42b6488722b8884f74b64fce8c396dc7 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:35:02 +0200 Subject: [PATCH 277/854] Update import statement for new module name --- nf_core/subworkflows/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 69e7983d9d..501cfa0424 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -11,7 +11,7 @@ from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME -from .subworkflow_command import SubworkflowCommand +from .subworkflows_command import SubworkflowCommand log = logging.getLogger(__name__) From 689cb8326080ebf9c042f6dfa6770113d7ead049 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:36:44 +0200 Subject: [PATCH 278/854] Add missing imports --- nf_core/components/components_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index f16b4f057f..ee9ee460ec 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -2,6 +2,9 @@ import os import nf_core.utils +import questionary +import rich.prompt + log = logging.getLogger(__name__) From 2d42ea59dc0c8e91efbd6713d6195b45ff5fac38 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:37:46 +0200 Subject: [PATCH 279/854] isort fixes --- nf_core/components/components.py | 4 ++-- nf_core/components/components_utils.py | 3 ++- nf_core/modules/modules_command.py | 1 - nf_core/subworkflows/subworkflows_command.py | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nf_core/components/components.py b/nf_core/components/components.py index a89472dcad..60b9d2776f 100644 --- a/nf_core/components/components.py +++ b/nf_core/components/components.py @@ -5,11 +5,11 @@ import yaml -from .components_utils import get_repo_type - from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo +from .components_utils import get_repo_type + log = logging.getLogger(__name__) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index ee9ee460ec..fe916c205c 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,10 +1,11 @@ import logging import os -import nf_core.utils import questionary import rich.prompt +import nf_core.utils + log = logging.getLogger(__name__) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 3bd176ef11..72fd9d2fea 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -3,7 +3,6 @@ from nf_core.components.components import Components - log = logging.getLogger(__name__) diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index 65de789057..b789d1f519 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -5,7 +5,6 @@ from nf_core.components.components import Components - log = logging.getLogger(__name__) From 485332423e7fba8175ce6d62183748c41fa37214 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:39:35 +0200 Subject: [PATCH 280/854] Prettier fixes --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ce16f65a9c..ce1e27f9ce 100644 --- a/README.md +++ b/README.md @@ -190,11 +190,13 @@ export NFCORE_NO_VERSION_CHECK=1 ### Update tools It is advisable to keep nf-core/tools updated to the most recent version. The command to update depends on the system used to install it, for example if you have installed it with conda you can use: + ```bash conda update nf-core ``` if you used pip: + ```bash pip install --upgrade nf-core ``` From a4b4645951ee6d0a7a377e518712df6a36f6f365 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 12 Oct 2022 10:52:58 +0200 Subject: [PATCH 281/854] make tests pass --- nf_core/modules/modules_json.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index e538f3ef27..bfeac885b1 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -482,8 +482,14 @@ def check_up_to_date(self): # If there are any modules/subworkflows left in 'modules.json' after all installed are removed, # we try to reinstall them if len(missing_installation) > 0: - resolve_missing_installation(self, missing_installation, "subworkflows") - resolve_missing_installation(self, missing_installation, "modules") + if "subworkflows" in [ + c_type for _, repo_content in missing_installation.items() for c_type in repo_content.keys() + ]: + self.resolve_missing_installation(missing_installation, "subworkflows") + if "modules" in [ + c_type for _, repo_content in missing_installation.items() for c_type in repo_content.keys() + ]: + self.resolve_missing_installation(missing_installation, "modules") # If some modules/subworkflows didn't have an entry in the 'modules.json' file # we try to determine the SHA from the commit log of the remote From 0dd78c48ce43b86a8610f00bdaf08da66deffbed Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 12 Oct 2022 09:45:23 +0000 Subject: [PATCH 282/854] [automated] Fix code linting --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ce16f65a9c..ce1e27f9ce 100644 --- a/README.md +++ b/README.md @@ -190,11 +190,13 @@ export NFCORE_NO_VERSION_CHECK=1 ### Update tools It is advisable to keep nf-core/tools updated to the most recent version. The command to update depends on the system used to install it, for example if you have installed it with conda you can use: + ```bash conda update nf-core ``` if you used pip: + ```bash pip install --upgrade nf-core ``` From 9cdc8afbc1dee3c6670d7803646cc5e8cf4c7077 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 12 Oct 2022 11:50:00 +0200 Subject: [PATCH 283/854] change class name to Component --- nf_core/components/components.py | 2 +- nf_core/modules/modules_command.py | 4 ++-- nf_core/subworkflows/subworkflows_command.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/components/components.py b/nf_core/components/components.py index 60b9d2776f..0eb7f3af63 100644 --- a/nf_core/components/components.py +++ b/nf_core/components/components.py @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) -class Components: +class Component: """ Base class for the 'nf-core modules' and 'nf-core subworkflows' commands """ diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 72fd9d2fea..ccf1f84e53 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -1,12 +1,12 @@ import logging from pathlib import Path -from nf_core.components.components import Components +from nf_core.components.components import Component log = logging.getLogger(__name__) -class ModuleCommand(Components): +class ModuleCommand(Component): """ Base class for the 'nf-core modules' commands """ diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index b789d1f519..f511abd608 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -3,12 +3,12 @@ import shutil from pathlib import Path -from nf_core.components.components import Components +from nf_core.components.components import Component log = logging.getLogger(__name__) -class SubworkflowCommand(Components): +class SubworkflowCommand(Component): """ Base class for the 'nf-core subworkflows' commands """ From eb4a158155ffc40b932004e386377587a1acd029 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 12 Oct 2022 11:51:42 +0200 Subject: [PATCH 284/854] actually change it com ComponentCommand --- nf_core/components/components.py | 2 +- nf_core/modules/modules_command.py | 4 ++-- nf_core/subworkflows/subworkflows_command.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/components/components.py b/nf_core/components/components.py index 0eb7f3af63..d81227a2ce 100644 --- a/nf_core/components/components.py +++ b/nf_core/components/components.py @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) -class Component: +class ComponentCommand: """ Base class for the 'nf-core modules' and 'nf-core subworkflows' commands """ diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index ccf1f84e53..94b7fa65fe 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -1,12 +1,12 @@ import logging from pathlib import Path -from nf_core.components.components import Component +from nf_core.components.components import ComponentCommand log = logging.getLogger(__name__) -class ModuleCommand(Component): +class ModuleCommand(ComponentCommand): """ Base class for the 'nf-core modules' commands """ diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index f511abd608..71f4b68b72 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -3,12 +3,12 @@ import shutil from pathlib import Path -from nf_core.components.components import Component +from nf_core.components.components import ComponentCommand log = logging.getLogger(__name__) -class SubworkflowCommand(Component): +class SubworkflowCommand(ComponentCommand): """ Base class for the 'nf-core subworkflows' commands """ From 92e3d60c66c1ff258933988c96d2670e3664b033 Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Wed, 12 Oct 2022 12:03:42 +0200 Subject: [PATCH 285/854] Update ci.yml --- nf_core/pipeline-template/.github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 64cc12a26d..f539fab335 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -11,6 +11,10 @@ on: env: NXF_ANSI_LOG: false +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + jobs: test: name: Run pipeline with test data From 56d57e10650c7b2e68f38d08e9b0603bf3b6ae78 Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Wed, 12 Oct 2022 12:21:48 +0200 Subject: [PATCH 286/854] add nf-core subworkflows info command --- nf_core/__main__.py | 39 +++- nf_core/modules/modules_repo.py | 18 ++ nf_core/subworkflows/__init__.py | 1 + nf_core/subworkflows/info.py | 335 +++++++++++++++++++++++++++++++ 4 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 nf_core/subworkflows/info.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 4e65efb155..be6cb8d4c6 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -58,7 +58,7 @@ "nf-core subworkflows": [ { "name": "For pipelines", - "commands": ["list", "install"], + "commands": ["list", "info", "install"], }, { "name": "Developing new subworkflows", @@ -1020,6 +1020,43 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin sys.exit(1) +# nf-core subworkflows info +@subworkflows.command() +@click.pass_context +@click.argument("tool", type=str, required=False, metavar=" or ") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: Current working directory][/]", +) +def info(ctx, tool, dir): + """ + Show developer usage information about a given subworkflow. + + Parses information from a subworkflow's [i]meta.yml[/] and renders help + on the command line. A handy equivalent to searching the + [link=https://nf-co.re/modules]nf-core website[/]. + + If run from a pipeline and a local copy of the subworkflow is found, the command + will print this usage info. + If not, usage from the remote subworkflows repo will be shown. + """ + try: + subworkflow_info = nf_core.subworkflows.SubworkflowInfo( + dir, + tool, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + stdout.print(subworkflow_info.get_subworkflow_info()) + except (UserWarning, LookupError) as e: + log.error(e) + sys.exit(1) + + # nf-core subworkflows install @subworkflows.command() @click.pass_context diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 81761cbcf5..bd3e827c92 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -561,3 +561,21 @@ def get_meta_yml(self, module_name): with open(path) as fh: contents = fh.read() return contents + + def get_subworkflow_meta_yml(self, subworkflow_name): + """ + Returns the contents of the 'meta.yml' file of a module + + Args: + subworkflow_name (str): The name of the subworkflow + + Returns: + (str): The contents of the file in text format + """ + self.checkout_branch() + path = os.path.join(self.subworkflows_dir, subworkflow_name, "meta.yml") + if not os.path.exists(path): + return None + with open(path) as fh: + contents = fh.read() + return contents diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index 437e0b02ec..c9ef546205 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,4 +1,5 @@ from .create import SubworkflowCreate +from .info import SubworkflowInfo from .install import SubworkflowInstall from .list import SubworkflowList from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py new file mode 100644 index 0000000000..3ad0d5c78e --- /dev/null +++ b/nf_core/subworkflows/info.py @@ -0,0 +1,335 @@ +import logging +import os +import shutil +from pathlib import Path + +import questionary +import yaml +from rich import box +from rich.console import Group +from rich.markdown import Markdown +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +import nf_core.modules.module_utils +from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.module_utils import get_repo_type +from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE, ModulesRepo + +log = logging.getLogger(__name__) + + +class SubworkflowInfo(object): + """ + Class to print information of a subworkflow. + + Attributes + ---------- + meta : YAML object + stores the information from meta.yml file + local_path : str + path of the local subworkflows + remote_location : str + remote repository URL + local : bool + indicates if the subworkflow is locally installed or not + repo_type : str + repository type. Can be either 'pipeline' or 'modules' + modules_json : ModulesJson object + contains 'modules.json' file information from a pipeline + module : str + name of the tool to get information from + + Methods + ------- + init_mod_name(subworkflow) + Makes sure that we have a subworkflow name + get_subworkflow_info() + Given the name of a subworkflow, parse meta.yml and print usage help + get_local_yaml() + Attempt to get the meta.yml file from a locally installed subworkflow + get_remote_yaml() + Attempt to get the meta.yml file from a remote repo + generate_subworkflow_info_help() + Take the parsed meta.yml and generate rich help + """ + + def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): + # super().__init__(pipeline_dir, remote_url, branch, no_pull) + self.dir = pipeline_dir + self.modules_repo = ModulesRepo(remote_url, branch, no_pull) + try: + if self.dir: + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + else: + self.repo_type = None + except LookupError as e: + raise UserWarning(e) + + self.meta = None + self.local_path = None + self.remote_location = None + self.local = None + + # Quietly check if this is a pipeline or not + if pipeline_dir: + try: + pipeline_dir, repo_type = get_repo_type(pipeline_dir, use_prompt=False) + log.debug(f"Found {repo_type} repo: {pipeline_dir}") + except UserWarning as e: + log.debug(f"Only showing remote info: {e}") + pipeline_dir = None + + if self.repo_type == "pipeline": + # # Check modules directory structure + # self.check_modules_structure() + # Check modules.json up to date + self.modules_json = ModulesJson(self.dir) + self.modules_json.check_up_to_date() + else: + self.modules_json = None + self.module = self.init_mod_name(tool) + + def init_mod_name(self, subworkflow): + """ + Makes sure that we have a subworkflow name before proceeding. + + Args: + module: str: Subworkflow name to check + """ + if subworkflow is None: + self.local = questionary.confirm( + "Is the subworkflow locally installed?", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + if self.local: + if self.repo_type == "modules": + subworkflows = self.get_subworkflows_clone_modules() + else: + subworkflows = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) + subworkflows = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in subworkflows] + if subworkflows is None: + raise UserWarning(f"No subworkflow installed from '{self.modules_repo.remote_url}'") + else: + subworkflows = self.modules_repo.get_avail_subworkflows() + subworkflow = questionary.autocomplete( + "Please select a subworkflow", choices=subworkflows, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + while subworkflow not in subworkflows: + log.info(f"'{subworkflow}' is not a valid subworkflow name") + subworkflow = questionary.autocomplete( + "Please select a new subworkflow", choices=subworkflows, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + + return subworkflow + + def get_subworkflow_info(self): + """Given the name of a subworkflow, parse meta.yml and print usage help.""" + + # Running with a local install, try to find the local meta + if self.local: + self.meta = self.get_local_yaml() + + # Either failed locally or in remote mode + if not self.meta: + self.meta = self.get_remote_yaml() + + # Could not find the meta + if self.meta is False: + raise UserWarning(f"Could not find subworkflow '{self.module}'") + + return self.generate_subworkflow_info_help() + + def get_local_yaml(self): + """Attempt to get the meta.yml file from a locally installed module. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + + if self.repo_type == "pipeline": + # Try to find and load the meta.yml file + repo_name = self.modules_repo.repo_path + module_base_path = os.path.join(self.dir, "subworkflows") + # Check that we have any modules installed from this repo + modules = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) + module_names = [module for _, module in modules] + if modules is None: + raise LookupError(f"No modules installed from {self.modules_repo.remote_url}") + + if self.module in module_names: + install_dir = [dir for dir, module in modules if module == self.module][0] + mod_dir = os.path.join(module_base_path, install_dir, self.module) + meta_fn = os.path.join(mod_dir, "meta.yml") + if os.path.exists(meta_fn): + log.debug(f"Found local file: {meta_fn}") + with open(meta_fn, "r") as fh: + self.local_path = mod_dir + return yaml.safe_load(fh) + + log.debug(f"Subworkflow '{self.module}' meta.yml not found locally") + else: + module_base_path = os.path.join(self.dir, "subworkflows", "nf-core") + if self.module in os.listdir(module_base_path): + mod_dir = os.path.join(module_base_path, self.module) + meta_fn = os.path.join(mod_dir, "meta.yml") + if os.path.exists(meta_fn): + log.debug(f"Found local file: {meta_fn}") + with open(meta_fn, "r") as fh: + self.local_path = mod_dir + return yaml.safe_load(fh) + log.debug(f"Subworkflow '{self.module}' meta.yml not found locally") + + return None + + def get_remote_yaml(self): + """Attempt to get the meta.yml file from a remote repo. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + # Check if our requested module is there + if self.module not in self.modules_repo.get_avail_subworkflows(): + return False + + file_contents = self.modules_repo.get_subworkflow_meta_yml(self.module) + if file_contents is None: + return False + self.remote_location = self.modules_repo.remote_url + return yaml.safe_load(file_contents) + + def generate_subworkflow_info_help(self): + """Take the parsed meta.yml and generate rich help. + + Returns: + rich renderable + """ + + renderables = [] + + # Intro panel + intro_text = Text() + if self.local_path: + intro_text.append(Text.from_markup(f"Location: [blue]{self.local_path}\n")) + elif self.remote_location: + intro_text.append( + Text.from_markup( + ":globe_with_meridians: Repository: " + f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" + f"{self.remote_location}" + f"{'[/link]' if self.remote_location.startswith('http') else '' }" + "\n" + ) + ) + + if self.meta.get("tools"): + tools_strings = [] + for tool in self.meta["tools"]: + for tool_name, tool_meta in tool.items(): + if "homepage" in tool_meta: + tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}[/link]") + else: + tools_strings.append(f"{tool_name}") + intro_text.append(Text.from_markup(f":wrench: Tools: {', '.join(tools_strings)}\n", style="dim")) + + if self.meta.get("description"): + intro_text.append(Text.from_markup(f":book: Description: {self.meta['description']}", style="dim")) + + renderables.append( + Panel( + intro_text, + title=f"[bold]Subworkflow: [green]{self.module}\n", + title_align="left", + ) + ) + + # Inputs + if self.meta.get("input"): + inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + inputs_table.add_column(":inbox_tray: Inputs") + inputs_table.add_column("Description") + inputs_table.add_column("Pattern", justify="right", style="green") + for input in self.meta["input"]: + for key, info in input.items(): + inputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + + renderables.append(inputs_table) + + # Outputs + if self.meta.get("output"): + outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + outputs_table.add_column(":outbox_tray: Outputs") + outputs_table.add_column("Description") + outputs_table.add_column("Pattern", justify="right", style="green") + for output in self.meta["output"]: + for key, info in output.items(): + outputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + + renderables.append(outputs_table) + + # Installation command + if self.remote_location: + cmd_base = "nf-core modules" + if self.remote_location != NF_CORE_MODULES_REMOTE: + cmd_base = f"nf-core modules --git-remote {self.remote_location}" + renderables.append( + Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.module}\n") + ) + + return Group(*renderables) + + # def check_modules_structure(self): + # """ + # Check that the structure of the modules directory in a pipeline is the correct one: + # modules/nf-core/TOOL/SUBTOOL + + # Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: + # modules/nf-core/modules/TOOL/SUBTOOL + # """ + # if self.repo_type == "pipeline": + # wrong_location_modules = [] + # for directory, _, files in os.walk(Path(self.dir, "modules")): + # if "main.nf" in files: + # module_path = Path(directory).relative_to(Path(self.dir, "modules")) + # parts = module_path.parts + # # Check that there are modules installed directly under the 'modules' directory + # if parts[1] == "modules": + # wrong_location_modules.append(module_path) + # # If there are modules installed in the wrong location + # if len(wrong_location_modules) > 0: + # log.info("The modules folder structure is outdated. Reinstalling modules.") + # # Remove the local copy of the modules repository + # log.info(f"Updating '{self.modules_repo.local_repo_dir}'") + # self.modules_repo.setup_local_repo( + # self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress + # ) + # # Move wrong modules to the right directory + # for module in wrong_location_modules: + # modules_dir = Path("modules").resolve() + # correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) + # wrong_dir = Path(modules_dir, module) + # shutil.move(wrong_dir, correct_dir) + # log.info(f"Moved {wrong_dir} to {correct_dir}.") + # shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) + # # Regenerate modules.json file + # modules_json = ModulesJson(self.dir) + # modules_json.check_up_to_date() + + def get_subworkflows_clone_modules(self): + """ + Get the subworkflows available in a clone of nf-core/modules + """ + module_base_path = Path(self.dir, Path("subworkflows", "nf-core")) + return [ + str(Path(dir).relative_to(module_base_path)) + for dir, _, files in os.walk(module_base_path) + if "main.nf" in files + ] From 39ffa4653c06303679065beeb48505c0d9b4f66f Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Wed, 12 Oct 2022 12:28:04 +0200 Subject: [PATCH 287/854] fix python-isort --- nf_core/subworkflows/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index 3ad0d5c78e..07a97bc413 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -13,8 +13,8 @@ from rich.text import Text import nf_core.modules.module_utils -from nf_core.modules.modules_json import ModulesJson from nf_core.modules.module_utils import get_repo_type +from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE, ModulesRepo log = logging.getLogger(__name__) From 2104b5d30c7b4027306e51da53248eb6f08f5c24 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 12 Oct 2022 12:32:28 +0200 Subject: [PATCH 288/854] rename components.py to components_command.py --- nf_core/components/{components.py => components_command.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nf_core/components/{components.py => components_command.py} (100%) diff --git a/nf_core/components/components.py b/nf_core/components/components_command.py similarity index 100% rename from nf_core/components/components.py rename to nf_core/components/components_command.py From d8806ebf0bd292c1d8896c8cc7665e2121ebf99a Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Wed, 12 Oct 2022 12:54:10 +0200 Subject: [PATCH 289/854] fix tests failing because of renamed subworkflows --- tests/subworkflows/list.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index d6f58db63f..ac3a9b2f84 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -12,7 +12,7 @@ def test_subworkflows_list_remote(self): console = Console(record=True) console.print(listed_swfs) output = console.export_text() - assert "bam_sort_samtools" in output + assert "bam_sort_stats_samtools" in output # def test_subworkflows_list_remote_gitlab(self): @@ -23,26 +23,26 @@ def test_subworkflows_list_remote(self): # console = Console(record=True) # console.print(listed_swfs) # output = console.export_text() -# assert "bam_sort_samtools" in output +# assert "bam_sort_stats_samtools" in output def test_subworkflows_install_and_list_pipeline(self): """Test listing locally installed subworkflows""" - self.swfs_install.install("align_bowtie2") + self.swfs_install.install("fastq_align_bowtie2") swfs_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_swfs = swfs_list.list_subworkflows() console = Console(record=True) console.print(listed_swfs) output = console.export_text() - assert "align_bowtie2" in output + assert "fastq_align_bowtie2" in output # def test_subworkflows_install_gitlab_and_list_pipeline(self): # """Test listing locally installed subworkflows""" -# self.swfs_install_gitlab.install("bam_sort_samtools") +# self.swfs_install_gitlab.install("bam_sort_stats_samtools") # swfs_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) # listed_swfs = swfs_list.list_subworkflows() # console = Console(record=True) # console.print(listed_swfs) # output = console.export_text() -# assert "bam_sort_samtools" in output +# assert "bam_sort_stats_samtools" in output From fa4ef7fdc3eb25eb8234e28a7667324ab818f955 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 12 Oct 2022 12:57:48 +0200 Subject: [PATCH 290/854] add check_patch_paths back to modules_command.py --- nf_core/modules/modules_command.py | 33 +++++++++++++++++++- nf_core/subworkflows/subworkflows_command.py | 2 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 94b7fa65fe..c2579497f9 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -1,7 +1,7 @@ import logging from pathlib import Path -from nf_core.components.components import ComponentCommand +from nf_core.components.components_command import ComponentCommand log = logging.getLogger(__name__) @@ -20,3 +20,34 @@ def get_local_modules(self): """ local_module_dir = Path(self.dir, "modules", "local") return [str(path.relative_to(local_module_dir)) for path in local_module_dir.iterdir() if path.suffix == ".nf"] + + def check_patch_paths(self, patch_path, module_name): + """ + Check that paths in patch files are updated to the new modules path + """ + if patch_path.exists(): + log.info(f"Modules {module_name} contains a patch file.") + rewrite = False + with open(patch_path, "r") as fh: + lines = fh.readlines() + for index, line in enumerate(lines): + # Check if there are old paths in the patch file and replace + if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line: + rewrite = True + lines[index] = line.replace( + f"modules/{self.modules_repo.repo_path}/modules/{module_name}/", + f"modules/{self.modules_repo.repo_path}/{module_name}/", + ) + if rewrite: + log.info(f"Updating paths in {patch_path}") + with open(patch_path, "w") as fh: + for line in lines: + fh.write(line) + # Update path in modules.json if the file is in the correct format + modules_json = ModulesJson(self.dir) + modules_json.load() + if modules_json.has_git_url_and_modules(): + modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ + self.modules_repo.repo_path + ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) + modules_json.dump() diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index 71f4b68b72..d3b730d5a0 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path -from nf_core.components.components import ComponentCommand +from nf_core.components.components_command import ComponentCommand log = logging.getLogger(__name__) From 2c10142bd73972c867c420fbae159bea0367799d Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Wed, 12 Oct 2022 12:58:56 +0200 Subject: [PATCH 291/854] Update nf_core/pipeline-template/.github/workflows/ci.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index f539fab335..67e50b44c4 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -12,7 +12,7 @@ env: NXF_ANSI_LOG: false concurrency: - group: ${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: From 7e997d61e0e50107324300066dc4b4065a5a7fed Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 12 Oct 2022 13:10:29 +0200 Subject: [PATCH 292/854] import ModulesJson --- nf_core/modules/modules_command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index c2579497f9..d47d050c38 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -2,6 +2,7 @@ from pathlib import Path from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) From c56bdae948bf523543ef1a0c453b56e29d7a92c2 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 13:18:58 +0200 Subject: [PATCH 293/854] Fix broken nested list comprehension --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index bfeac885b1..3515bde2a4 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -160,7 +160,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): repos[nrepo_remote]["modules"][nrepo_name] = {} dirs_not_covered = self.dir_tree_uncovered( - modules_dir, [Path(name) for name in repos[url][modules_dir] for url in repos] + modules_dir, [Path(name) for url in repos for name in repos[url][modules_dir]] ) return repos, renamed_dirs From fb8fd60d534191cfaadf1ae75e948a4e7fd5e7c6 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 12 Oct 2022 13:15:31 +0200 Subject: [PATCH 294/854] Improve test documentation [skip ci] --- tests/test_sync.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 9d6eb5f488..09e4510431 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -176,7 +176,16 @@ def test_create_merge_base_branch(self): assert psync.merge_branch in psync.repo.branches def test_create_merge_base_branch_thrice(self): - """Try creating a merge base branch thrice""" + """Try creating a merge base branch thrice + + This is needed because the first time this function is called, the + merge branch does not exist yet (it is only created at the end of the + create_merge_base_branch function) and the if-statement is ignored. + Also, the second time this function is called, the existing merge + branch only has the base format, i.e. without the -{branch_no} at the + end, so it is needed to call it a third time to make sure this is + picked up. + """ psync = nf_core.sync.PipelineSync(self.pipeline_dir) psync.inspect_sync_dir() psync.get_wf_config() From 852be9095985dd33743bbf4b5780f8e58eb16e1d Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 12 Oct 2022 13:36:21 +0200 Subject: [PATCH 295/854] add docker_m1 profile --- CHANGELOG.md | 1 + nf_core/pipeline-template/nextflow.config | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f87f48ac7c..190f83b61e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Fix lint warnings for `samplesheet_check.nf` module +- Add profile for running `docker` with the Apple M1 chips. ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 737341c4b0..5aee7a6875 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -107,6 +107,14 @@ profiles { shifter.enabled = false charliecloud.enabled = false } + docker_m1 { + docker.enabled = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + } singularity { singularity.enabled = true singularity.autoMounts = true From 9264be86f452f75c2fa6e1e54d6701306b0317cf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 12 Oct 2022 11:57:58 +0000 Subject: [PATCH 296/854] [automated] Fix code linting --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 67e50b44c4..20a4711432 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: env: NXF_ANSI_LOG: false -concurrency: +concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true From e59b9c11f3ecce0703dc15584fff1deadc212e90 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 14:11:40 +0200 Subject: [PATCH 297/854] Prettier fixes --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 67e50b44c4..20a4711432 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: env: NXF_ANSI_LOG: false -concurrency: +concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true From 9ffdac0b85bc502de8ad495a1fd17a1752f6eaa9 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 12 Oct 2022 14:28:05 +0200 Subject: [PATCH 298/854] Fix broken jinja2 template caused by {{}} not wrapped in raw block --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 20a4711432..03282a2924 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -12,7 +12,7 @@ env: NXF_ANSI_LOG: false concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: "{% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}{% endraw %}" cancel-in-progress: true jobs: From 175f921bb2f220390477ca75c82df68af65839a5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 12 Oct 2022 14:49:02 +0200 Subject: [PATCH 299/854] fix critical bug in template --- nf_core/pipeline-template/.github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 67e50b44c4..03282a2924 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -11,8 +11,8 @@ on: env: NXF_ANSI_LOG: false -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +concurrency: + group: "{% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}{% endraw %}" cancel-in-progress: true jobs: From 157eafcd07eb9ea5356b617968713eb3728e8136 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 12 Oct 2022 16:38:03 +0200 Subject: [PATCH 300/854] fix(template): Update ci jobs to use nodejs16 https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/ --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 64cc12a26d..31585afdc9 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 9fb569ab0d..b2cf30270f 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -12,7 +12,7 @@ jobs: EditorConfig: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v2 @@ -25,7 +25,7 @@ jobs: Prettier: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v2 @@ -38,7 +38,7 @@ jobs: PythonBlack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Check code lints with Black uses: psf/black@stable @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 From 6422cecdc63834e6907fb0ddcf6e62e68fa3f602 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 12 Oct 2022 16:42:00 +0200 Subject: [PATCH 301/854] chore: Update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8fc3f1003..bd830c36d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ - Lint pyproject.toml file exists and content ([#1795](https://github.com/nf-core/tools/pull/1795)) - Update GitHub PyPI package release action to v1 ([#1785](https://github.com/nf-core/tools/pull/1785)) +### Template + +- Update GitHub actions to use nodejs16 ([#1944](https://github.com/nf-core/tools/pull/1944)) + ## [v2.5 - Gold Otter](https://github.com/nf-core/tools/releases/tag/2.5) - [2022-08-30] ### Template From 6bd773fada282fad66ff5b739829173a5b361b8c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Wed, 12 Oct 2022 16:45:50 +0200 Subject: [PATCH 302/854] ci: Update to use nodejs16 as well --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/lint-code.yml | 8 ++++---- .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/push_dockerhub_release.yml | 2 +- .github/workflows/pytest.yml | 2 +- .github/workflows/sync.yml | 4 ++-- .github/workflows/tools-api-docs-dev.yml | 2 +- .github/workflows/tools-api-docs-release.yml | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 277baf1425..5c93c1c784 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -21,7 +21,7 @@ jobs: - "latest-everything" steps: # Get the repo code - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Check out source-code repository # Set up nf-core/tools diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 6b2116d2f4..c2d1eb70bd 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -20,7 +20,7 @@ jobs: - "21.10.3" - "latest-everything" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Check out source-code repository - name: Set up Python 3.7 diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 8d951b416d..352ee56b20 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Check out source-code repository - name: Set up Python 3.7 diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index af2d41aecf..ee71772b55 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -10,7 +10,7 @@ jobs: EditorConfig: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v2 @@ -24,7 +24,7 @@ jobs: Prettier: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v2 @@ -37,7 +37,7 @@ jobs: PythonBlack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Check code lints with Black uses: psf/black@stable @@ -68,7 +68,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out source-code repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v2 diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index 88efe88b9d..dea28cdd35 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 71245244d8..dc82951ab6 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0828d93315..795ebcc789 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -18,7 +18,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Check out source-code repository - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 2d79807a0b..c95ef5e910 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -36,10 +36,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Check out nf-core/tools - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 8192c93ef2..2b1099daa3 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Check out source-code repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.7 uses: actions/setup-python@v3 diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 6dca273742..689be22869 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -19,7 +19,7 @@ jobs: - ${{ github.event.release.tag_name }} steps: - name: Check out source-code repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.7 uses: actions/setup-python@v3 From b3798e004afc09f608cb32210c780660cf39cf10 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 14 Oct 2022 15:53:38 +0100 Subject: [PATCH 303/854] set conda.enabled=true for Conda --- nf_core/pipeline-template/nextflow.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 737341c4b0..99acd8a0af 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -84,6 +84,7 @@ profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { params.enable_conda = true + conda.enabled = true docker.enabled = false singularity.enabled = false podman.enabled = false @@ -92,6 +93,7 @@ profiles { } mamba { params.enable_conda = true + conda.enabled = true conda.useMamba = true docker.enabled = false singularity.enabled = false From fba33008b090c2dbb7e0d1b368344433b2b6443b Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 17 Oct 2022 10:38:17 +0100 Subject: [PATCH 304/854] We weren't passing through the component type --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index d81227a2ce..fad48b6322 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -22,7 +22,7 @@ def __init__(self, component_type, dir, remote_url=None, branch=None, no_pull=Fa """ Initialise the ComponentClass object """ - self.component_type = "" + self.component_type = component_type self.dir = dir self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress = hide_progress From 81192538b182119074108c039c033a13fb7f2c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 17 Oct 2022 15:54:20 +0200 Subject: [PATCH 305/854] Apply suggestions from code review --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c3aaaeccd7..d139286ebe 100644 --- a/README.md +++ b/README.md @@ -421,7 +421,7 @@ The interactive prompts will guide you through the pipeline creation process. An name: coolpipe description: A cool pipeline author: me -prefix: cool-pipes-company +prefix: myorg skip: - github - ci @@ -430,7 +430,7 @@ skip: - nf_core_configs ``` -This will create a pipeline called `coolpipe` in the directory `cool-pipes-company-coolpipe` (`-`) with `me` as the author. It will exclude all possible parts of the template: +This will create a pipeline called `coolpipe` in the directory `myorg-coolpipe` (`-`) with `me` as the author. It will exclude all possible parts of the template: - `github`: removed all files required for GitHub hosting of the pipeline. Specifically, the `.github` folder and `.gitignore` file. - `ci`: removes the GitHub continuous integration tests from the pipeline. Specifically, the `.github/workflows/` folder. From 765ed752177219afb4dd69525b3dc12fbc33a617 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Tue, 18 Oct 2022 21:08:37 +0000 Subject: [PATCH 306/854] Add codespaces template for creating new repo's. --- .../.devcontainer/Dockerfile | 21 +++++++ .../.devcontainer/add-notice.sh | 20 +++++++ .../.devcontainer/devcontainer.json | 55 +++++++++++++++++++ .../pipeline-template/.devcontainer/noop.txt | 3 + 4 files changed, 99 insertions(+) create mode 100644 nf_core/pipeline-template/.devcontainer/Dockerfile create mode 100644 nf_core/pipeline-template/.devcontainer/add-notice.sh create mode 100644 nf_core/pipeline-template/.devcontainer/devcontainer.json create mode 100644 nf_core/pipeline-template/.devcontainer/noop.txt diff --git a/nf_core/pipeline-template/.devcontainer/Dockerfile b/nf_core/pipeline-template/.devcontainer/Dockerfile new file mode 100644 index 0000000000..85ad388646 --- /dev/null +++ b/nf_core/pipeline-template/.devcontainer/Dockerfile @@ -0,0 +1,21 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda/.devcontainer/base.Dockerfile + +FROM mcr.microsoft.com/vscode/devcontainers/anaconda:0-3 + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="none" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# Copy environment.yml (if found) to a temp location so we update the environment. Also +# copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists. +COPY environment.yml* .devcontainer/noop.txt /tmp/conda-tmp/ +RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; fi \ + && rm -rf /tmp/conda-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +RUN conda install -c bioconda nextflow +RUN pip install nf-core + diff --git a/nf_core/pipeline-template/.devcontainer/add-notice.sh b/nf_core/pipeline-template/.devcontainer/add-notice.sh new file mode 100644 index 0000000000..0417abb6cd --- /dev/null +++ b/nf_core/pipeline-template/.devcontainer/add-notice.sh @@ -0,0 +1,20 @@ +# Display a notice when not running in GitHub Codespaces + +cat << 'EOF' > /usr/local/etc/vscode-dev-containers/conda-notice.txt +When using "conda" from outside of GitHub Codespaces, note the Anaconda repository +contains restrictions on commercial use that may impact certain organizations. See +https://aka.ms/vscode-remote/conda/anaconda + +EOF + +notice_script="$(cat << 'EOF' +if [ -t 1 ] && [ "${IGNORE_NOTICE}" != "true" ] && [ "${TERM_PROGRAM}" = "vscode" ] && [ "${CODESPACES}" != "true" ] && [ ! -f "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed" ]; then + cat "/usr/local/etc/vscode-dev-containers/conda-notice.txt" + mkdir -p "$HOME/.config/vscode-dev-containers" + ((sleep 10s; touch "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed") &) +fi +EOF +)" + +echo "${notice_script}" | tee -a /etc/bash.bashrc >> /etc/zsh/zshrc + diff --git a/nf_core/pipeline-template/.devcontainer/devcontainer.json b/nf_core/pipeline-template/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..c82f6703a5 --- /dev/null +++ b/nf_core/pipeline-template/.devcontainer/devcontainer.json @@ -0,0 +1,55 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda +{ + "name": "Anaconda (Python 3)", + "build": { + "context": "..", + "dockerfile": "Dockerfile", + "args": { + "NODE_VERSION": "lts/*" + } + }, + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/opt/conda/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", + "python.formatting.yapfPath": "/opt/conda/bin/yapf", + "python.linting.flake8Path": "/opt/conda/bin/flake8", + "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", + "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", + "python.linting.pylintPath": "/opt/conda/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ] + } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "python --version", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + "features": { + "docker-in-docker": "20.10", + "git": "os-provided", + "ghcr.io/devcontainers/features/conda:1": {}, + "ghcr.io/devcontainers/features/docker-from-docker:1": {}, + "ghcr.io/devcontainers/features/git:1": {} + } +} + + diff --git a/nf_core/pipeline-template/.devcontainer/noop.txt b/nf_core/pipeline-template/.devcontainer/noop.txt new file mode 100644 index 0000000000..09c5ae9b86 --- /dev/null +++ b/nf_core/pipeline-template/.devcontainer/noop.txt @@ -0,0 +1,3 @@ +This file copied into the container along with environment.yml* from the parent +folder. This file is included to prevents the Dockerfile COPY instruction from +failing if no environment.yml is found. From c2a2e913362368fce7ce337b3fa179b411dbad5b Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Tue, 18 Oct 2022 21:13:45 +0000 Subject: [PATCH 307/854] Update change log. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b67ffeaff2..12c6888ce2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Fix lint warnings for `samplesheet_check.nf` module +- Add codespaces template ([#1957](https://github.com/nf-core/tools/pull/1957)) ### Linting From f9722a6d2b3a32a4d8a9606ef43f5d8efe345b40 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Tue, 18 Oct 2022 21:17:12 +0000 Subject: [PATCH 308/854] Update readme with codepsaces. --- nf_core/pipeline-template/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 02a32f1f6e..590d27b7db 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -73,6 +73,17 @@ The results obtained from the full-sized test can be viewed on the [nf-core webs {% if branded -%} +## Codespaces +This repo includes a devcontainer configuration which will create a GitHub Codespace for Nextflow development! + +Devcontainer specs: +- [DevContainer config](.devcontainer/devcontainer.json) +- [Dockerfile](.devcontainer/Dockerfile) + +# Getting started +- Create a new repo in GitHub using this [template](https://github.com/openwdl workflow-template-wdl/generate). +- Open the repo in [Codespaces](../../codespaces) + ## Documentation The {{ name }} pipeline comes with documentation about the pipeline [usage](https://nf-co.re/{{ short_name }}/usage), [parameters](https://nf-co.re/{{ short_name }}/parameters) and [output](https://nf-co.re/{{ short_name }}/output). From 20ca392c6b3c7876686d871c0d2028aa41bcd758 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Tue, 18 Oct 2022 16:26:44 -0500 Subject: [PATCH 309/854] Update README.md --- nf_core/pipeline-template/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 590d27b7db..fa45b56229 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -81,8 +81,10 @@ Devcontainer specs: - [Dockerfile](.devcontainer/Dockerfile) # Getting started -- Create a new repo in GitHub using this [template](https://github.com/openwdl workflow-template-wdl/generate). - Open the repo in [Codespaces](../../codespaces) +- Tools installed + - nf-core + - nextflow ## Documentation From 838e2b8765a4e48b37836d22d092dc490a4a24a3 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 15 Oct 2022 17:24:07 +0200 Subject: [PATCH 310/854] use pytest raises instead of try --- tests/test_schema.py | 56 +++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 4f829875e7..8bd7e88b24 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -179,11 +179,9 @@ def test_validate_schema_fail_duplicate_ids(self): "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"foo": {}}}}, "allOf": [{"$ref": "#/definitions/groupOne"}, {"$ref": "#/definitions/groupTwo"}], } - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - raise UserWarning("Expected AssertionError") - except AssertionError as e: - assert e.args[0] == "Duplicate parameter found in schema `definitions`: `foo`" + assert exc_info.value.args[0] == "Duplicate parameter found in schema `definitions`: `foo`" def test_validate_schema_fail_missing_def(self): """ @@ -193,11 +191,9 @@ def test_validate_schema_fail_missing_def(self): "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, "allOf": [{"$ref": "#/definitions/groupOne"}], } - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - raise UserWarning("Expected AssertionError") - except AssertionError as e: - assert e.args[0] == "Definition subschema `groupTwo` not included in schema `allOf`" + assert exc_info.value.args[0] == "Definition subschema `groupTwo` not included in schema `allOf`" def test_validate_schema_fail_unexpected_allof(self): """ @@ -211,11 +207,9 @@ def test_validate_schema_fail_unexpected_allof(self): {"$ref": "#/definitions/groupThree"}, ], } - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - raise UserWarning("Expected AssertionError") - except AssertionError as e: - assert e.args[0] == "Subschema `groupThree` found in `allOf` but not `definitions`" + assert exc_info.value.args[0] == "Subschema `groupThree` found in `allOf` but not `definitions`" def test_make_skeleton_schema(self): """Test making a new schema skeleton""" @@ -383,20 +377,17 @@ def __init__(self, data, status_code): def test_launch_web_builder_404(self, mock_post): """Mock launching the web builder""" self.schema_obj.web_schema_build_url = "invalid_url" - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.launch_web_builder() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - assert e.args[0] == "Could not access remote API results: invalid_url (HTML 404 Error)" + assert exc_info.value.args[0] == "Could not access remote API results: invalid_url (HTML 404 Error)" @mock.patch("requests.post", side_effect=mocked_requests_post) def test_launch_web_builder_invalid_status(self, mock_post): """Mock launching the web builder""" self.schema_obj.web_schema_build_url = "valid_url_error" - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.launch_web_builder() - except AssertionError as e: - assert e.args[0].startswith("Pipeline schema builder response not recognised") + assert exc_info.value.args[0].startswith("Pipeline schema builder response not recognised") @mock.patch("requests.post", side_effect=mocked_requests_post) @mock.patch("requests.get") @@ -404,12 +395,10 @@ def test_launch_web_builder_invalid_status(self, mock_post): def test_launch_web_builder_success(self, mock_post, mock_get, mock_webbrowser): """Mock launching the web builder""" self.schema_obj.web_schema_build_url = "valid_url_success" - try: + # Assertion error comes from the get_web_builder_response() function + with pytest.raises(AssertionError) as exc_info: self.schema_obj.launch_web_builder() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - # Assertion error comes from get_web_builder_response() function - assert e.args[0].startswith("Could not access remote API results: https://nf-co.re") + assert exc_info.value.args[0].startswith("Could not access remote API results: https://nf-co.re") def mocked_requests_get(*args, **kwargs): """Helper function to emulate GET requests responses from the web""" @@ -438,21 +427,17 @@ def __init__(self, data, status_code): def test_get_web_builder_response_404(self, mock_post): """Mock launching the web builder""" self.schema_obj.web_schema_build_api_url = "invalid_url" - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.get_web_builder_response() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - assert e.args[0] == "Could not access remote API results: invalid_url (HTML 404 Error)" + assert exc_info.value.args[0] == "Could not access remote API results: invalid_url (HTML 404 Error)" @mock.patch("requests.get", side_effect=mocked_requests_get) def test_get_web_builder_response_error(self, mock_post): """Mock launching the web builder""" self.schema_obj.web_schema_build_api_url = "valid_url_error" - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.get_web_builder_response() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - assert e.args[0] == "Got error from schema builder: 'testing URL failure'" + assert exc_info.value.args[0] == "Got error from schema builder: 'testing URL failure'" @mock.patch("requests.get", side_effect=mocked_requests_get) def test_get_web_builder_response_waiting(self, mock_post): @@ -464,10 +449,7 @@ def test_get_web_builder_response_waiting(self, mock_post): def test_get_web_builder_response_saved(self, mock_post): """Mock launching the web builder""" self.schema_obj.web_schema_build_api_url = "valid_url_saved" - try: + with pytest.raises(AssertionError) as exc_info: self.schema_obj.get_web_builder_response() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - # Check that this is the expected AssertionError, as there are several - assert e.args[0].startswith("Response from schema builder did not pass validation") + assert exc_info.value.args[0].startswith("Response from schema builder did not pass validation") assert self.schema_obj.schema == {"foo": "bar"} From 74faf8b10caa19fb2ff5c91d7eb0d11256d24fc1 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 15 Oct 2022 17:29:34 +0200 Subject: [PATCH 311/854] use pytest raises instead of try --- tests/test_launch.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tests/test_launch.py b/tests/test_launch.py index a438f98c2f..c7ff8bdd7c 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -8,6 +8,8 @@ import unittest from unittest import mock +import pytest + import nf_core.create import nf_core.launch @@ -135,11 +137,9 @@ def test_launch_web_gui_missing_keys(self, mock_poll_nfcore_web_api): """Check the code that opens the web browser""" self.launcher.get_pipeline_schema() self.launcher.merge_nxf_flag_schema() - try: + with pytest.raises(AssertionError) as exc_info: self.launcher.launch_web_gui() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - assert e.args[0].startswith("Web launch response not recognised:") + assert exc_info.value.args[0].startswith("Web launch response not recognised:") @mock.patch( "nf_core.utils.poll_nfcore_web_api", side_effect=[{"api_url": "foo", "web_url": "bar", "status": "recieved"}] @@ -155,20 +155,16 @@ def test_launch_web_gui(self, mock_poll_nfcore_web_api, mock_webbrowser, mock_wa @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "error", "message": "foo"}]) def test_get_web_launch_response_error(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - status error""" - try: + with pytest.raises(AssertionError) as exc_info: self.launcher.get_web_launch_response() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - assert e.args[0] == "Got error from launch API (foo)" + assert exc_info.value.args[0] == "Got error from launch API (foo)" @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "foo"}]) def test_get_web_launch_response_unexpected(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - status error""" - try: + with pytest.raises(AssertionError) as exc_info: self.launcher.get_web_launch_response() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - assert e.args[0].startswith("Web launch GUI returned unexpected status (foo): ") + assert exc_info.value.args[0].startswith("Web launch GUI returned unexpected status (foo): ") @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "waiting_for_user"}]) def test_get_web_launch_response_waiting(self, mock_poll_nfcore_web_api): @@ -178,11 +174,9 @@ def test_get_web_launch_response_waiting(self, mock_poll_nfcore_web_api): @mock.patch("nf_core.utils.poll_nfcore_web_api", side_effect=[{"status": "launch_params_complete"}]) def test_get_web_launch_response_missing_keys(self, mock_poll_nfcore_web_api): """Test polling the website for a launch response - complete, but missing keys""" - try: + with pytest.raises(AssertionError) as exc_info: self.launcher.get_web_launch_response() - raise UserWarning("Should have hit an AssertionError") - except AssertionError as e: - assert e.args[0] == "Missing return key from web API: 'nxf_flags'" + assert exc_info.value.args[0] == "Missing return key from web API: 'nxf_flags'" @mock.patch( "nf_core.utils.poll_nfcore_web_api", From 7f0a59efe51c7756f23e7ed2e546b52c8add7564 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 15 Oct 2022 17:56:57 +0200 Subject: [PATCH 312/854] use pathlib.Path.touch instead of open close --- tests/test_sync.py | 13 +++++++------ tests/test_utils.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 09e4510431..da8249b1e0 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -7,6 +7,7 @@ import shutil import tempfile import unittest +from pathlib import Path from unittest import mock import git @@ -47,8 +48,8 @@ def test_inspect_sync_dir_notgit(self, tmp_dir): def test_inspect_sync_dir_dirty(self): """Try syncing a pipeline with uncommitted changes""" # Add an empty file, uncommitted - test_fn = os.path.join(self.pipeline_dir, "uncommitted") - open(test_fn, "a").close() + test_fn = Path(self.pipeline_dir) / "uncommitted" + test_fn.touch() # Try to sync, check we halt with the right error psync = nf_core.sync.PipelineSync(self.pipeline_dir) try: @@ -140,8 +141,8 @@ def test_commit_template_changes_changes(self): psync.get_wf_config() psync.checkout_template_branch() # Add an empty file, uncommitted - test_fn = os.path.join(self.pipeline_dir, "uncommitted") - open(test_fn, "a").close() + test_fn = Path(self.pipeline_dir) / "uncommitted" + test_fn.touch() # Check that we have uncommitted changes assert psync.repo.is_dirty(untracked_files=True) is True # Function returns True if no changes were made @@ -157,8 +158,8 @@ def test_push_template_branch_error(self): psync.get_wf_config() psync.checkout_template_branch() # Add an empty file and commit it - test_fn = os.path.join(self.pipeline_dir, "uncommitted") - open(test_fn, "a").close() + test_fn = Path(self.pipeline_dir) / "uncommitted" + test_fn.touch() psync.commit_template_changes() # Try to push changes with pytest.raises(nf_core.sync.PullRequestException) as exc_info: diff --git a/tests/test_utils.py b/tests/test_utils.py index f914d675a5..b43d6e8ff2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -120,7 +120,7 @@ def test_list_files_no_git(self, tmpdir): """Test listing pipeline files without `git-ls`""" # Create a test file in a temporary directory tmp_fn = os.path.join(tmpdir, "testfile") - open(tmp_fn, "a").close() + Path(tmp_fn).touch() pipeline_obj = nf_core.utils.Pipeline(tmpdir) pipeline_obj._list_files() assert tmp_fn in pipeline_obj.files From 92100b68691a3154c8f3bc5532cac6e3de14c591 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 15 Oct 2022 18:25:38 +0200 Subject: [PATCH 313/854] remove needless else --- tests/test_sync.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index da8249b1e0..458b268245 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -241,7 +241,7 @@ def json(self): if url == os.path.join(url_template.format("no_existing_pr"), "pulls?head=TEMPLATE&base=None"): response_data = [] return MockResponse(response_data, 200) - elif url == os.path.join(url_template.format("list_prs"), "pulls"): + if url == os.path.join(url_template.format("list_prs"), "pulls"): response_data = [ { "state": "closed", @@ -350,8 +350,7 @@ def test_close_open_template_merge_prs(self, mock_get): for pr in prs: if pr["state"] != "open": continue - else: - mock_close_open_pr.assert_any_call(pr) + mock_close_open_pr.assert_any_call(pr) @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) From aa4596df68452b891a1a4bb1825b4b457c35577b Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 18 Oct 2022 23:48:18 +0200 Subject: [PATCH 314/854] more expressive if block Co-authored-by: Adrien Coulier --- tests/test_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 458b268245..7d5158b74f 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -348,8 +348,8 @@ def test_close_open_template_merge_prs(self, mock_get): prs = mock_get(f"https://api.github.com/repos/{psync.gh_repo}/pulls").data for pr in prs: - if pr["state"] != "open": - continue + if pr["state"] == "open": + mock_close_open_pr.assert_any_call(pr) mock_close_open_pr.assert_any_call(pr) @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) From 3aec7345d843481f6701303b4cd5462a78700801 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 19 Oct 2022 00:01:25 +0200 Subject: [PATCH 315/854] remove duplicated assert_any_call Co-authored-by: Adrien Coulier --- tests/test_sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 7d5158b74f..4460b56054 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -350,7 +350,6 @@ def test_close_open_template_merge_prs(self, mock_get): for pr in prs: if pr["state"] == "open": mock_close_open_pr.assert_any_call(pr) - mock_close_open_pr.assert_any_call(pr) @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) From 492d6f506850f5cb91cf0ef93a7c15119f1691ef Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Wed, 19 Oct 2022 11:02:08 -0500 Subject: [PATCH 316/854] Change to absolute url --- nf_core/pipeline-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index fa45b56229..8eceb4816f 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -81,7 +81,7 @@ Devcontainer specs: - [Dockerfile](.devcontainer/Dockerfile) # Getting started -- Open the repo in [Codespaces](../../codespaces) +- Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) - Tools installed - nf-core - nextflow From c1de099d8d0854b55459c7ef0c7038301127de40 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 19 Oct 2022 23:24:13 +0200 Subject: [PATCH 317/854] enable setting defaulBranch for pipeline repo creation --- nf_core/create.py | 16 ++++++++++------ tests/test_sync.py | 5 +++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 8b028d90af..9f9503e879 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -49,6 +49,7 @@ def __init__( author, version="1.0dev", no_git=False, + default_branch=None, force=False, outdir=None, template_yaml_path=None, @@ -82,6 +83,7 @@ def __init__( self.no_git = ( no_git if self.template_params["github"] else True ) # Set to True if template was configured without github hosting + self.default_branch = default_branch self.force = force if outdir is None: outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) @@ -525,19 +527,21 @@ def git_init_pipeline(self): """ # Check that the default branch is not dev try: - default_branch = git.config.GitConfigParser().get_value("init", "defaultBranch") + default_branch = self.default_branch or git.config.GitConfigParser().get_value("init", "defaultBranch") except configparser.Error: default_branch = None log.debug("Could not read init.defaultBranch") - if default_branch == "dev" or default_branch == "TEMPLATE": + if default_branch in ["dev", "TEMPLATE"]: raise UserWarning( - f"Your Git defaultBranch is set to '{default_branch}', which is incompatible with nf-core.\n" - "This can be modified with the command [white on grey23] git config --global init.defaultBranch [/]\n" - "Pipeline git repository is not initialised." + f"Your Git defaultBranch '{default_branch}' is incompatible with nf-core.\n" + "Set the default branch name with " + "[white on grey23] git config --global init.defaultBranch [/]\n" + # Or set the default_branch parameter in this class + "Pipeline git repository will not be initialised." ) # Initialise pipeline log.info("Initialising pipeline git repository") - repo = git.Repo.init(self.outdir) + repo = git.Repo.init(self.outdir, initial_branch=default_branch) repo.git.add(A=True) repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") # Add TEMPLATE branch to git repository diff --git a/tests/test_sync.py b/tests/test_sync.py index 4460b56054..cf5d772f1c 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -26,12 +26,13 @@ def setUp(self): """Create a new pipeline to test""" self.tmp_dir = tempfile.mkdtemp() self.pipeline_dir = os.path.join(self.tmp_dir, "test_pipeline") + default_branch = "master" self.create_obj = nf_core.create.PipelineCreate( - "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True + "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True, default_branch=default_branch ) self.create_obj.init_pipeline() self.remote_path = os.path.join(self.tmp_dir, "remote_repo") - self.remote_repo = git.Repo.init(self.remote_path, bare=True) + self.remote_repo = git.Repo.init(self.remote_path, bare=True, initial_branch=default_branch) def tearDown(self): if os.path.exists(self.tmp_dir): From 6dd98742a626d54db41e51c609d202c666fd8acc Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 19 Oct 2022 23:37:14 +0200 Subject: [PATCH 318/854] the default branch can now be specified for new pipeline repos --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf016b8e63..25ef314297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) - Improve test coverage of sync.py - `check_up_to_date()` function from `modules_json` also checks for subworkflows. +- The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). ### Modules From 50aa97430d573e8bcdfdeecf99d853c8a30c5a96 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 19 Oct 2022 23:54:55 +0200 Subject: [PATCH 319/854] only set the initial branch if default banch is given --- nf_core/create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 9f9503e879..eb664ad4ec 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -541,7 +541,8 @@ def git_init_pipeline(self): ) # Initialise pipeline log.info("Initialising pipeline git repository") - repo = git.Repo.init(self.outdir, initial_branch=default_branch) + init_kwargs = {"initial_branch": default_branch} if default_branch is not None else {} + repo = git.Repo.init(self.outdir, **init_kwargs) repo.git.add(A=True) repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") # Add TEMPLATE branch to git repository From f19288766d0d79952464b01d56e17f3efcb7aae0 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 08:47:17 +0200 Subject: [PATCH 320/854] move the default_branch to the end of the argument list --- nf_core/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index eb664ad4ec..78aa54ed36 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -49,11 +49,11 @@ def __init__( author, version="1.0dev", no_git=False, - default_branch=None, force=False, outdir=None, template_yaml_path=None, plain=False, + default_branch=None, ): self.template_params, skip_paths_keys = self.create_param_dict( name, description, author, version, template_yaml_path, plain From fe5fb7b7bd8e7222169236d7527b9c08fb41ba15 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 08:55:11 +0200 Subject: [PATCH 321/854] add docstrings for git related arguments --- nf_core/create.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/create.py b/nf_core/create.py index 78aa54ed36..183877303f 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -40,6 +40,8 @@ class PipelineCreate(object): May the force be with you. outdir (str): Path to the local output directory. template_yaml (str): Path to template.yml file for pipeline creation settings. + plain (bool): If true the Git repository will be initialized plain. + default_branch (str): Specifies the --initial-branch name. """ def __init__( From c3fe2e52d37e0d977e14303a9fa3a3d3fb5e43d2 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 09:36:41 +0200 Subject: [PATCH 322/854] use git branch rename as git<2.28 lacks the initial-branch argument --- nf_core/create.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 183877303f..31caa44fde 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -543,8 +543,9 @@ def git_init_pipeline(self): ) # Initialise pipeline log.info("Initialising pipeline git repository") - init_kwargs = {"initial_branch": default_branch} if default_branch is not None else {} - repo = git.Repo.init(self.outdir, **init_kwargs) + repo = git.Repo.init(self.outdir) + if default_branch: + repo.active_branch.rename(default_branch) repo.git.add(A=True) repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") # Add TEMPLATE branch to git repository From 83b7eaf0cfb665430db5fe39e6da67f8677e32b5 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 09:59:43 +0200 Subject: [PATCH 323/854] clean up comments --- nf_core/create.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 31caa44fde..8fef3fb83b 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -527,7 +527,6 @@ def git_init_pipeline(self): Raises: UserWarning: if Git default branch is set to 'dev' or 'TEMPLATE'. """ - # Check that the default branch is not dev try: default_branch = self.default_branch or git.config.GitConfigParser().get_value("init", "defaultBranch") except configparser.Error: @@ -538,17 +537,16 @@ def git_init_pipeline(self): f"Your Git defaultBranch '{default_branch}' is incompatible with nf-core.\n" "Set the default branch name with " "[white on grey23] git config --global init.defaultBranch [/]\n" - # Or set the default_branch parameter in this class + "Or set the default_branch parameter in this class.\n" "Pipeline git repository will not be initialised." ) - # Initialise pipeline + log.info("Initialising pipeline git repository") repo = git.Repo.init(self.outdir) if default_branch: repo.active_branch.rename(default_branch) repo.git.add(A=True) repo.index.commit(f"initial template build from nf-core/tools, version {nf_core.__version__}") - # Add TEMPLATE branch to git repository repo.git.branch("TEMPLATE") repo.git.branch("dev") log.info( From 9cda484a251d0e8d472f28429e6a5fb4419111bd Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 10:01:28 +0200 Subject: [PATCH 324/854] simplify code --- nf_core/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 8fef3fb83b..6367b397c4 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -527,10 +527,10 @@ def git_init_pipeline(self): Raises: UserWarning: if Git default branch is set to 'dev' or 'TEMPLATE'. """ + default_branch = self.default_branch try: - default_branch = self.default_branch or git.config.GitConfigParser().get_value("init", "defaultBranch") + default_branch = git.config.GitConfigParser().get_value("init", "defaultBranch") except configparser.Error: - default_branch = None log.debug("Could not read init.defaultBranch") if default_branch in ["dev", "TEMPLATE"]: raise UserWarning( From b0a013d2f35dc232cd67b82c979c3baf25c95d2d Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 10:09:54 +0200 Subject: [PATCH 325/854] restore precedence for default_branch --- nf_core/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 6367b397c4..1166c1b0f6 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -529,7 +529,7 @@ def git_init_pipeline(self): """ default_branch = self.default_branch try: - default_branch = git.config.GitConfigParser().get_value("init", "defaultBranch") + default_branch = default_branch or git.config.GitConfigParser().get_value("init", "defaultBranch") except configparser.Error: log.debug("Could not read init.defaultBranch") if default_branch in ["dev", "TEMPLATE"]: From 0004b9b8808d3f405c8dde6a90accd9ca49c74db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 20 Oct 2022 11:02:45 +0200 Subject: [PATCH 326/854] back to use Workflow name --- nf_core/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index c3bc73870b..457adc3266 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -203,7 +203,7 @@ def get_param(self, param_name, passed_value, template_yaml, template_yaml_path) return passed_value def prompt_wf_name(self): - wf_name = questionary.text("Name", style=nf_core.utils.nfcore_question_style).unsafe_ask() + wf_name = questionary.text("Workflow name", style=nf_core.utils.nfcore_question_style).unsafe_ask() while not re.match(r"^[a-z]+$", wf_name): log.error("[red]Invalid workflow name: must be lowercase without punctuation.") wf_name = questionary.text( From 66dbd4fec07ad572ae2153da4b3d7cfb9cd5e92b Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 11:29:33 +0200 Subject: [PATCH 327/854] add test for defaul_branch argument in pipeline creation --- tests/test_create.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_create.py b/tests/test_create.py index 5f8f6546f2..5f732fc180 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -4,6 +4,8 @@ import os import unittest +import git + import nf_core.create from .utils import with_temporary_folder @@ -16,6 +18,7 @@ def setUp(self, tmp_path): self.pipeline_description = "just for 4w3s0m3 tests" self.pipeline_author = "Chuck Norris" self.pipeline_version = "1.0.0" + self.default_branch = "default" self.pipeline = nf_core.create.PipelineCreate( name=self.pipeline_name, @@ -26,6 +29,7 @@ def setUp(self, tmp_path): force=True, outdir=tmp_path, plain=True, + default_branch=self.default_branch, ) def test_pipeline_creation(self): @@ -37,3 +41,4 @@ def test_pipeline_creation(self): def test_pipeline_creation_initiation(self): self.pipeline.init_pipeline() assert os.path.isdir(os.path.join(self.pipeline.outdir, ".git")) + assert f" {self.default_branch}\n" in git.Repo.init(self.pipeline.outdir).git.branch() From 9b7348c9315f064a3b1de2f9ec2fc3edf5f74732 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 13:15:46 +0200 Subject: [PATCH 328/854] list reserved branch names in error message --- nf_core/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/create.py b/nf_core/create.py index 1166c1b0f6..8124ea21b2 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -535,6 +535,7 @@ def git_init_pipeline(self): if default_branch in ["dev", "TEMPLATE"]: raise UserWarning( f"Your Git defaultBranch '{default_branch}' is incompatible with nf-core.\n" + "'dev' and 'TEMPLATE' can not be used as default branch name.\n" "Set the default branch name with " "[white on grey23] git config --global init.defaultBranch [/]\n" "Or set the default_branch parameter in this class.\n" From 3bbc5583aa7d5c316b4b70e8b43ac028cb3557e9 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 20 Oct 2022 13:22:12 +0200 Subject: [PATCH 329/854] initial-branch is not available on git<2.28 --- tests/test_sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index cf5d772f1c..b968b64370 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -32,7 +32,8 @@ def setUp(self): ) self.create_obj.init_pipeline() self.remote_path = os.path.join(self.tmp_dir, "remote_repo") - self.remote_repo = git.Repo.init(self.remote_path, bare=True, initial_branch=default_branch) + self.remote_repo = git.Repo.init(self.remote_path, bare=True) + self.remote_repo.active_branch.rename(default_branch) def tearDown(self): if os.path.exists(self.tmp_dir): From ca69a9fd10ff8a516ea2ae12cd10f7d22324016c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Langer?= <61791748+bjlang@users.noreply.github.com> Date: Thu, 20 Oct 2022 15:31:05 +0200 Subject: [PATCH 330/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/__main__.py | 2 +- nf_core/modules/modules_repo.py | 2 +- nf_core/subworkflows/info.py | 6 +++--- nf_core/subworkflows/list.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 08ef5c8c4a..689c1a1155 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1023,7 +1023,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin # nf-core subworkflows info @subworkflows.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index e8daa0a03b..ed1f3c8703 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -560,7 +560,7 @@ def get_meta_yml(self, module_name): def get_subworkflow_meta_yml(self, subworkflow_name): """ - Returns the contents of the 'meta.yml' file of a module + Returns the contents of the 'meta.yml' file of a subworkflow Args: subworkflow_name (str): The name of the subworkflow diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index 07a97bc413..eaf0adbe7c 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -96,7 +96,7 @@ def init_mod_name(self, subworkflow): Makes sure that we have a subworkflow name before proceeding. Args: - module: str: Subworkflow name to check + subworkflow: str: Subworkflow name to check """ if subworkflow is None: self.local = questionary.confirm( @@ -277,9 +277,9 @@ def generate_subworkflow_info_help(self): # Installation command if self.remote_location: - cmd_base = "nf-core modules" + cmd_base = "nf-core subworkflows" if self.remote_location != NF_CORE_MODULES_REMOTE: - cmd_base = f"nf-core modules --git-remote {self.remote_location}" + cmd_base = f"nf-core subworkflows --git-remote {self.remote_location}" renderables.append( Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.module}\n") ) diff --git a/nf_core/subworkflows/list.py b/nf_core/subworkflows/list.py index ae1dc5790c..71eff5fdfd 100644 --- a/nf_core/subworkflows/list.py +++ b/nf_core/subworkflows/list.py @@ -88,7 +88,7 @@ def pattern_msg(keywords): # Verify that 'modules.json' is consistent with the installed modules modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() # TODO: check subworkflows also!!!! + modules_json.check_up_to_date() # Filter by keywords repos_with_swfs = { repo_url: [swf for swf in subworkflows if all(k in swf[1] for k in keywords)] From 817f0c9addd4a236409e82ff91571bcd1492aa49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Langer?= <61791748+bjlang@users.noreply.github.com> Date: Thu, 20 Oct 2022 15:48:00 +0200 Subject: [PATCH 331/854] Apply suggestions from code review --- nf_core/subworkflows/info.py | 75 +++++++++--------------------------- 1 file changed, 19 insertions(+), 56 deletions(-) diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index eaf0adbe7c..042cb9f0e8 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -89,7 +89,7 @@ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): self.modules_json.check_up_to_date() else: self.modules_json = None - self.module = self.init_mod_name(tool) + self.subworkflow = self.init_mod_name(tool) def init_mod_name(self, subworkflow): """ @@ -136,7 +136,7 @@ def get_subworkflow_info(self): # Could not find the meta if self.meta is False: - raise UserWarning(f"Could not find subworkflow '{self.module}'") + raise UserWarning(f"Could not find subworkflow '{self.subworkflow}'") return self.generate_subworkflow_info_help() @@ -151,15 +151,15 @@ def get_local_yaml(self): # Try to find and load the meta.yml file repo_name = self.modules_repo.repo_path module_base_path = os.path.join(self.dir, "subworkflows") - # Check that we have any modules installed from this repo - modules = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - module_names = [module for _, module in modules] - if modules is None: - raise LookupError(f"No modules installed from {self.modules_repo.remote_url}") - - if self.module in module_names: - install_dir = [dir for dir, module in modules if module == self.module][0] - mod_dir = os.path.join(module_base_path, install_dir, self.module) + # Check that we have any subworkflows installed from this repo + subworkflows = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) + subworkflow_names = [subworkflow for _, subworkflow in subworkflows] + if subworkflows is None: + raise LookupError(f"No subworkflows installed from {self.modules_repo.remote_url}") + + if self.subworkflow in subworkflow_names: + install_dir = [dir for dir, subworkflow in subworkflows if subworkflow == self.subworkflow][0] + mod_dir = os.path.join(module_base_path, install_dir, self.subworkflow) meta_fn = os.path.join(mod_dir, "meta.yml") if os.path.exists(meta_fn): log.debug(f"Found local file: {meta_fn}") @@ -167,18 +167,18 @@ def get_local_yaml(self): self.local_path = mod_dir return yaml.safe_load(fh) - log.debug(f"Subworkflow '{self.module}' meta.yml not found locally") + log.debug(f"Subworkflow '{self.subworkflow}' meta.yml not found locally") else: module_base_path = os.path.join(self.dir, "subworkflows", "nf-core") - if self.module in os.listdir(module_base_path): - mod_dir = os.path.join(module_base_path, self.module) + if self.subworkflow in os.listdir(module_base_path): + mod_dir = os.path.join(module_base_path, self.subworkflow) meta_fn = os.path.join(mod_dir, "meta.yml") if os.path.exists(meta_fn): log.debug(f"Found local file: {meta_fn}") with open(meta_fn, "r") as fh: self.local_path = mod_dir return yaml.safe_load(fh) - log.debug(f"Subworkflow '{self.module}' meta.yml not found locally") + log.debug(f"Subworkflow '{self.subworkflow}' meta.yml not found locally") return None @@ -189,10 +189,10 @@ def get_remote_yaml(self): dict or bool: Parsed meta.yml found, False otherwise """ # Check if our requested module is there - if self.module not in self.modules_repo.get_avail_subworkflows(): + if self.subworkflow not in self.modules_repo.get_avail_subworkflows(): return False - file_contents = self.modules_repo.get_subworkflow_meta_yml(self.module) + file_contents = self.modules_repo.get_subworkflow_meta_yml(self.subworkflow) if file_contents is None: return False self.remote_location = self.modules_repo.remote_url @@ -238,7 +238,7 @@ def generate_subworkflow_info_help(self): renderables.append( Panel( intro_text, - title=f"[bold]Subworkflow: [green]{self.module}\n", + title=f"[bold]Subworkflow: [green]{self.subworkflow}\n", title_align="left", ) ) @@ -281,48 +281,11 @@ def generate_subworkflow_info_help(self): if self.remote_location != NF_CORE_MODULES_REMOTE: cmd_base = f"nf-core subworkflows --git-remote {self.remote_location}" renderables.append( - Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.module}\n") + Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.subworkflow}\n") ) return Group(*renderables) - # def check_modules_structure(self): - # """ - # Check that the structure of the modules directory in a pipeline is the correct one: - # modules/nf-core/TOOL/SUBTOOL - - # Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: - # modules/nf-core/modules/TOOL/SUBTOOL - # """ - # if self.repo_type == "pipeline": - # wrong_location_modules = [] - # for directory, _, files in os.walk(Path(self.dir, "modules")): - # if "main.nf" in files: - # module_path = Path(directory).relative_to(Path(self.dir, "modules")) - # parts = module_path.parts - # # Check that there are modules installed directly under the 'modules' directory - # if parts[1] == "modules": - # wrong_location_modules.append(module_path) - # # If there are modules installed in the wrong location - # if len(wrong_location_modules) > 0: - # log.info("The modules folder structure is outdated. Reinstalling modules.") - # # Remove the local copy of the modules repository - # log.info(f"Updating '{self.modules_repo.local_repo_dir}'") - # self.modules_repo.setup_local_repo( - # self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress - # ) - # # Move wrong modules to the right directory - # for module in wrong_location_modules: - # modules_dir = Path("modules").resolve() - # correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) - # wrong_dir = Path(modules_dir, module) - # shutil.move(wrong_dir, correct_dir) - # log.info(f"Moved {wrong_dir} to {correct_dir}.") - # shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) - # # Regenerate modules.json file - # modules_json = ModulesJson(self.dir) - # modules_json.check_up_to_date() - def get_subworkflows_clone_modules(self): """ Get the subworkflows available in a clone of nf-core/modules From df2d4a08c9b3804386025b77d61d220b9d489baf Mon Sep 17 00:00:00 2001 From: fabianegli Date: Fri, 21 Oct 2022 09:09:06 +0200 Subject: [PATCH 332/854] remove unused argument since --- nf_core/modules/modules_repo.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 85e4a5b573..bd16e55000 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -443,17 +443,14 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 commits = iter(commits_new + commits_old) return commits - def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): + def get_subworkflow_git_log(self, subworkflow_name, depth=None): """ - Fetches the commit history the of requested subworkflow since a given date. The default value is - not arbitrary - it is the last time the structure of the nf-core/subworkflow repository was had an - update breaking backwards compatibility. + Fetches the commit history the of requested subworkflow. Args: subworkflow_name (str): Name of subworkflow modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question per_page (int): Number of commits per page returned by API page_nbr (int): Page number of the retrieved commits - since (str): Only show commits later than this timestamp. Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. Returns: From 237e628be7d28db71c0e96be007399452e1232cd Mon Sep 17 00:00:00 2001 From: fabianegli Date: Fri, 21 Oct 2022 09:09:51 +0200 Subject: [PATCH 333/854] update docstring of get_subworkflow_git_log --- nf_core/modules/modules_repo.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index bd16e55000..72c794d85e 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -448,10 +448,7 @@ def get_subworkflow_git_log(self, subworkflow_name, depth=None): Fetches the commit history the of requested subworkflow. Args: subworkflow_name (str): Name of subworkflow - modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question - per_page (int): Number of commits per page returned by API - page_nbr (int): Page number of the retrieved commits - Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. + depth (int): Maximal number of commits Returns: ( dict ): Iterator of commit SHAs and associated (truncated) message From f3160d096368ae2b3118562276b5248011f59907 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Fri, 21 Oct 2022 09:15:05 +0200 Subject: [PATCH 334/854] code maintenance --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e1afb421..bfa422e8c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,8 @@ - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) - Improve test coverage of sync.py - `check_up_to_date()` function from `modules_json` also checks for subworkflows. -- The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). +- The default branch can now be specified when creating a new pipeline repo ([#1959](https://github.com/nf-core/tools/pull/1959)) +- Code maintenance ([#1818](https://github.com/nf-core/tools/pull/1818)) ### Modules @@ -37,7 +38,7 @@ - Update MultiQC module, update supplying MultiQC default and custom config and logo files to module - Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) - Fix template spacing modified by JINJA ([#1830](https://github.com/nf-core/tools/pull/1830)) -- Fix MultiQC execution on template [#1855](https://github.com/nf-core/tools/pull/1855) +- Fix MultiQC execution on template ([#1855](https://github.com/nf-core/tools/pull/1855)) - Don't skip including `base.config` when skipping nf-core/configs ### Linting From 4bd77c4dba60210bb2ffc440d8d3c7d4eb49e97f Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:37:02 +0100 Subject: [PATCH 335/854] Add initial draft of subworkflows test command --- nf_core/subworkflows/subworkflows_test.py | 182 ++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 nf_core/subworkflows/subworkflows_test.py diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py new file mode 100644 index 0000000000..2e0f614a99 --- /dev/null +++ b/nf_core/subworkflows/subworkflows_test.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +""" +The SubworkflowsTest class runs the tests locally +""" + +import logging +import os +import sys +from pathlib import Path +from shutil import which + +import pytest +import questionary +import rich + +import nf_core.modules.module_utils +import nf_core.utils +from nf_core.modules.modules_json import ModulesJson + +from .subworkflows_command import SubworkflowCommand + +log = logging.getLogger(__name__) + + +class SubworkflowsTest(SubworkflowCommand): + """ + Class to run module pytests. + + ... + + Attributes + ---------- + module_name : str + name of the tool to run tests for + no_prompts : bool + flat indicating if prompts are used + pytest_args : tuple + additional arguments passed to pytest command + + Methods + ------- + run(): + Run test steps + _check_inputs(): + Check inputs. Ask for module_name if not provided and check that the directory exists + _set_profile(): + Set software profile + _run_pytests(self): + Run pytest + """ + + def __init__( + self, + subworkflow_name=None, + no_prompts=False, + pytest_args="", + remote_url=None, + branch=None, + no_pull=False, + ): + self.subworkflow_name = subworkflow_name + self.no_prompts = no_prompts + self.pytest_args = pytest_args + + super().__init__(".", remote_url, branch, no_pull) + + def run(self): + """Run test steps""" + if not self.no_prompts: + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + ) + self._check_inputs() + self._set_profile() + self._check_profile() + self._run_pytests() + + def _check_inputs(self): + """Do more complex checks about supplied flags.""" + # Check modules directory structure + self.check_modules_structure() + + # Retrieving installed modules + if self.repo_type == "modules": + installed_subwf = self.get_subworkflows_clone_subworkflows() + else: + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() + installed_subwf = modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) + + # Get the tool name if not specified + if self.module_name is None: + if self.no_prompts: + raise UserWarning( + "Subworkflow name not provided and prompts deactivated. Please provide the Subworkflow name SUBWORKFLOW." + ) + if not installed_subwf: + raise UserWarning( + f"No installed subworkflows were found from '{self.modules_repo.remote_url}'.\n" + f"Are you running the tests inside the nf-core/modules main directory?\n" + f"Otherwise, make sure that the directory structure is subworkflows/SUBWORKFLOW/ and tests/subworkflows/SUBWORKFLOW/" + ) + self.subworkflow_name = questionary.autocomplete( + "Subworkflow name:", + choices=installed_subwf, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Sanity check that the module directory exists + self._validate_folder_structure() + + def _validate_folder_structure(self): + """Validate that the modules follow the correct folder structure to run the tests: + - subworkflows/nf-core/SUBWORKFLOW/ + - tests/subworkflows/nf-core/SUBWORKFLOW/ + + """ + subworkflow_path = Path(self.default_subworkflows_path) / self.subworkflow_name + test_path = Path(self.default_subworkflows_tests_path) / self.subworkflow_name + + if not (self.dir / subworkflow_path).is_dir(): + raise UserWarning( + f"Cannot find directory '{subworkflow_path}'. Should be SUBWORKFLOW. Are you running the tests inside the nf-core/modules main directory?" + ) + if not (self.dir / test_path).is_dir(): + raise UserWarning( + f"Cannot find directory '{test_path}'. Should be SUBWORKFLOW. " + "Are you running the tests inside the nf-core/modules main directory? " + "Do you have tests for the specified module?" + ) + + def _set_profile(self): + """Set $PROFILE env variable. + The config expects $PROFILE and Nextflow fails if it's not set. + """ + if os.environ.get("PROFILE") is None: + os.environ["PROFILE"] = "" + if self.no_prompts: + log.info( + "Setting environment variable '$PROFILE' to an empty string as not set.\n" + "Tests will run with Docker by default. " + "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." + ) + else: + question = { + "type": "list", + "name": "profile", + "message": "Choose software profile", + "choices": ["Docker", "Singularity", "Conda"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + profile = answer["profile"].lower() + os.environ["PROFILE"] = profile + log.info(f"Setting environment variable '$PROFILE' to '{profile}'") + + def _check_profile(self): + """Check if profile is available""" + profile = os.environ.get("PROFILE") + # Make sure the profile read from the environment is a valid Nextflow profile. + valid_nextflow_profiles = ["docker", "singularity", "conda"] + if profile in valid_nextflow_profiles: + if not which(profile): + raise UserWarning(f"Command '{profile}' not found - is it installed?") + else: + raise UserWarning( + f"The PROFILE '{profile}' set in the shell environment is not valid.\n" + f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." + ) + + def _run_pytests(self): + """Given a subworkflow name, run tests.""" + # Print nice divider line + console = rich.console.Console() + console.rule(self.subworkflow_name, style="black") + + # Set pytest arguments + command_args = ["--tag", f"{self.subworkflow_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] + command_args += self.pytest_args + + # Run pytest + log.info(f"Running pytest for module '{self.subworkflow_name}'") + sys.exit(pytest.main(command_args)) From 6c075d236ccf0fd6bcdddc8a1a2a0fefd2205c47 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Oct 2022 12:57:21 +0200 Subject: [PATCH 336/854] add vestions.yml to create-test-yml but not md5sim --- nf_core/modules/test_yml_builder.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index c6ba65f161..23d7cc0e31 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -232,9 +232,6 @@ def create_test_file_dict(self, results_dir, is_repeat=False): test_files = [] for root, _, files in os.walk(results_dir, followlinks=True): for filename in files: - # Check that the file is not versions.yml - if filename == "versions.yml": - continue file_path = os.path.join(root, filename) # add the key here so that it comes first in the dict test_file = {"path": file_path} @@ -245,8 +242,10 @@ def create_test_file_dict(self, results_dir, is_repeat=False): # Add the md5 anyway, linting should fail later and can be manually removed if needed. # Originally we skipped this if empty, but then it's too easy to miss the warning. # Equally, if a file is legitimately empty we don't want to prevent this from working. - file_md5 = self._md5(file_path) - test_file["md5sum"] = file_md5 + if filename != "versions.yml": + # Only add md5sum if the file is not versions.yml + file_md5 = self._md5(file_path) + test_file["md5sum"] = file_md5 # Switch out the results directory path with the expected 'output' directory test_file["path"] = file_path.replace(results_dir, "output") test_files.append(test_file) From daf5da9a34630aba3e0c1019a1371deec0f5b904 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Oct 2022 13:18:38 +0200 Subject: [PATCH 337/854] only warn valid directory if pipeline is not nf-core --- nf_core/modules/modules_command.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 83705a0bb7..af361718a4 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -64,7 +64,9 @@ def has_valid_directory(self): main_nf = os.path.join(self.dir, "main.nf") nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + if Path(self.dir).resolve().parts[-1].startswith("nf-core"): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") return True def has_modules_file(self): From 5aac1567b65eb3704fad5f1491a290a9dcd7e8fd Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Oct 2022 13:19:50 +0200 Subject: [PATCH 338/854] Revert "only warn valid directory if pipeline is not nf-core" This reverts commit daf5da9a34630aba3e0c1019a1371deec0f5b904. --- nf_core/modules/modules_command.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index af361718a4..83705a0bb7 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -64,9 +64,7 @@ def has_valid_directory(self): main_nf = os.path.join(self.dir, "main.nf") nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): - if Path(self.dir).resolve().parts[-1].startswith("nf-core"): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") - log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") return True def has_modules_file(self): From a09e2e9d19a6f7fcbe55ce4c020f6d5490abe1c2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Oct 2022 13:20:55 +0200 Subject: [PATCH 339/854] only warn valid directory if pipeline is not nf-core --- nf_core/modules/modules_command.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 83705a0bb7..af361718a4 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -64,7 +64,9 @@ def has_valid_directory(self): main_nf = os.path.join(self.dir, "main.nf") nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + if Path(self.dir).resolve().parts[-1].startswith("nf-core"): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") return True def has_modules_file(self): From 8b30598a273022a37d21127a7fef903d5c84a15b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Oct 2022 13:23:14 +0200 Subject: [PATCH 340/854] modify changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e1afb421..def135025b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Improve test coverage of sync.py - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). +- Only warn when checking that the pipeline directory contains a `main.nf` and a `nextflow.config` file if the pipeline is not an nf-core pipeline [#1964](https://github.com/nf-core/tools/pull/1964) ### Modules From cbaecc2e89a3a87cfaf40be837e047adc2593c7c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Oct 2022 13:26:09 +0200 Subject: [PATCH 341/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e1afb421..89ffec54be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Improve test coverage of sync.py - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). +- Add file `versions.yml` when generating `test.yml` with `nf-core modules create-test-yml` but don't check for md5sum [#1963](https://github.com/nf-core/tools/pull/1963) ### Modules From a79918ef50b7d7a368dd590a0835f1354c4c0816 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Oct 2022 15:06:23 +0200 Subject: [PATCH 342/854] fix test --- tests/modules/install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/modules/install.py b/tests/modules/install.py index 06c5173346..7b7d86c2d7 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -22,7 +22,8 @@ def test_modules_install_nopipeline(self): @with_temporary_folder def test_modules_install_emptypipeline(self, tmpdir): """Test installing a module - empty dir given""" - self.mods_install.dir = tmpdir + os.mkdir(os.path.join(tmpdir, "nf-core-pipe")) + self.mods_install.dir = os.path.join(tmpdir, "nf-core-pipe") with pytest.raises(UserWarning) as excinfo: self.mods_install.install("foo") assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) From 65a134a66e15da4ffb1ebda00aa2bd64d745365f Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 24 Oct 2022 14:44:26 +0000 Subject: [PATCH 343/854] Remove redudant mixing of MQC version --- nf_core/pipeline-template/workflows/pipeline.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 370cbb77c0..582f47c4a0 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -107,7 +107,6 @@ workflow {{ short_name|upper }} { ch_multiqc_logo.toList() ) multiqc_report = MULTIQC.out.report.toList() - ch_versions = ch_versions.mix(MULTIQC.out.versions) } /* From 38ed7183db23a7bfc708580c8ae9736f84f60a0b Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 24 Oct 2022 14:45:45 +0000 Subject: [PATCH 344/854] Change unique operator to filter on version file contents --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 582f47c4a0..b39148308b 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -82,7 +82,7 @@ workflow {{ short_name|upper }} { ch_versions = ch_versions.mix(FASTQC.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') + ch_versions.unique{ it.text }.collectFile(name: 'collated_versions.yml') ) // From ea8c2e61aea5becf41c4ef83b02740658718ee56 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 10:11:35 +0200 Subject: [PATCH 345/854] mock biocontainers and anaconda api calls --- requirements-dev.txt | 1 + tests/test_modules.py | 26 +++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 011cbcc3c4..8fc8d1c7de 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ pytest-cov pytest-datafiles Sphinx sphinx_rtd_theme +requests_mock diff --git a/tests/test_modules.py b/tests/test_modules.py index a4e777a623..52a7a71bd7 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -7,6 +7,9 @@ import tempfile import unittest +import requests +import requests_mock + import nf_core.create import nf_core.modules @@ -32,9 +35,26 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n"]) - # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) - module_create.create() + # mock biocontainers response + with requests_mock.Mocker() as mock: + biocontainers_api_url = f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/bpipe/versions/bpipe-0.9.11" + anaconda_api_url = f"https://api.anaconda.org/package/bioconda/bpipe" + mock.register_uri("GET", biocontainers_api_url, text="to modify when the api works and I can know what to add") + anaconda_mock = { + "status_code": 200, + "latest_version": "0.9.11", + "summary": "", + "doc_url": "", + "dev_url": "", + "files": [{"version": "0.9.11"}], + "license": "", + } + biocontainers_mock = {"status_code": 200, "images": [{"image_type": "Docker", "image_name": ""}]} + mock.register_uri("GET", anaconda_api_url, json=anaconda_mock) + mock.register_uri("GET", biocontainers_api_url, json=biocontainers_mock) + # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) + module_create.create() return root_dir From 689fd8f21fc8e108e984d4d58cd73d8247251e73 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 10:43:28 +0200 Subject: [PATCH 346/854] mock api calls in all create tests --- CHANGELOG.md | 1 + tests/modules/create.py | 42 ++++++++++++++++++++++++++--------------- tests/test_modules.py | 20 +++----------------- tests/utils.py | 19 +++++++++++++++++++ 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index def135025b..17a6e1e5b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). - Only warn when checking that the pipeline directory contains a `main.nf` and a `nextflow.config` file if the pipeline is not an nf-core pipeline [#1964](https://github.com/nf-core/tools/pull/1964) +- Mock biocontainers and anaconda api calls in module tests [#1966](https://github.com/nf-core/tools/pull/1966) ### Modules diff --git a/tests/modules/create.py b/tests/modules/create.py index db7a66fa5c..61a8777b14 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,43 +1,55 @@ import os import pytest +import requests_mock import nf_core.modules +from tests.utils import mock_api_calls def test_modules_create_succeed(self): """Succeed at creating the TrimGalore! module""" - module_create = nf_core.modules.ModuleCreate( - self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" - ) - module_create.create() + with requests_mock.Mocker() as mock: + mock_api_calls(mock, "trim-galore", "0.6.7") + module_create = nf_core.modules.ModuleCreate( + self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" + ) + module_create.create() assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" - module_create = nf_core.modules.ModuleCreate( - self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" - ) - module_create.create() - with pytest.raises(UserWarning) as excinfo: + with requests_mock.Mocker() as mock: + mock_api_calls(mock, "trim-galore", "0.6.7") + module_create = nf_core.modules.ModuleCreate( + self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" + ) module_create.create() + with pytest.raises(UserWarning) as excinfo: + module_create.create() assert "Module file exists already" in str(excinfo.value) def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" - module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "fastqc", "@author", "process_low", False, False) - module_create.create() + with requests_mock.Mocker() as mock: + mock_api_calls(mock, "fastqc", "0.11.9") + module_create = nf_core.modules.ModuleCreate( + self.nfcore_modules, "fastqc", "@author", "process_low", False, False + ) + module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "fastqc", "main.nf")) assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "fastqc", "main.nf")) def test_modules_create_nfcore_modules_subtool(self): """Create a tool/subtool module in a nf-core/modules clone""" - module_create = nf_core.modules.ModuleCreate( - self.nfcore_modules, "star/index", "@author", "process_medium", False, False - ) - module_create.create() + with requests_mock.Mocker() as mock: + mock_api_calls(mock, "star", "2.8.10a") + module_create = nf_core.modules.ModuleCreate( + self.nfcore_modules, "star/index", "@author", "process_medium", False, False + ) + module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "main.nf")) assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "star", "index", "main.nf")) diff --git a/tests/test_modules.py b/tests/test_modules.py index 52a7a71bd7..157de39979 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -7,7 +7,6 @@ import tempfile import unittest -import requests import requests_mock import nf_core.create @@ -20,6 +19,7 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, + mock_api_calls, ) @@ -35,23 +35,9 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n"]) - # mock biocontainers response + # mock biocontainers and anaconda response with requests_mock.Mocker() as mock: - biocontainers_api_url = f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/bpipe/versions/bpipe-0.9.11" - anaconda_api_url = f"https://api.anaconda.org/package/bioconda/bpipe" - mock.register_uri("GET", biocontainers_api_url, text="to modify when the api works and I can know what to add") - anaconda_mock = { - "status_code": 200, - "latest_version": "0.9.11", - "summary": "", - "doc_url": "", - "dev_url": "", - "files": [{"version": "0.9.11"}], - "license": "", - } - biocontainers_mock = {"status_code": 200, "images": [{"image_type": "Docker", "image_name": ""}]} - mock.register_uri("GET", anaconda_api_url, json=anaconda_mock) - mock.register_uri("GET", biocontainers_api_url, json=biocontainers_mock) + mock_api_calls(mock, "bpipe", "0.9.11") # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) module_create.create() diff --git a/tests/utils.py b/tests/utils.py index fb592a6209..8f12a2cc7e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -66,3 +66,22 @@ def set_wd(path: Path): yield finally: os.chdir(start_wd) + + +def mock_api_calls(mock, module, version): + """Mock biocontainers and anaconda api calls for module""" + biocontainers_api_url = f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version}" + anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" + mock.register_uri("GET", biocontainers_api_url, text="to modify when the api works and I can know what to add") + anaconda_mock = { + "status_code": 200, + "latest_version": version, + "summary": "", + "doc_url": "", + "dev_url": "", + "files": [{"version": version}], + "license": "", + } + biocontainers_mock = {"status_code": 200, "images": [{"image_type": "Docker", "image_name": ""}]} + mock.register_uri("GET", anaconda_api_url, json=anaconda_mock) + mock.register_uri("GET", biocontainers_api_url, json=biocontainers_mock) From 572f1e83852d768b5a8cd71029e95769641efc66 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 10:51:47 +0200 Subject: [PATCH 347/854] mock also subworkflows api calls --- CHANGELOG.md | 2 +- tests/test_subworkflows.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17a6e1e5b5..055369e2b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). - Only warn when checking that the pipeline directory contains a `main.nf` and a `nextflow.config` file if the pipeline is not an nf-core pipeline [#1964](https://github.com/nf-core/tools/pull/1964) -- Mock biocontainers and anaconda api calls in module tests [#1966](https://github.com/nf-core/tools/pull/1966) +- Mock biocontainers and anaconda api calls in modules and subworkflows tests [#1966](https://github.com/nf-core/tools/pull/1966) ### Modules diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index e5d3f7d3f4..de52ed589c 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -7,11 +7,13 @@ import tempfile import unittest +import requests_mock + import nf_core.create import nf_core.modules import nf_core.subworkflows -from .utils import GITLAB_URL +from .utils import GITLAB_URL, mock_api_calls def create_modules_repo_dummy(tmp_dir): @@ -29,9 +31,11 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n"]) - # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) - module_create.create() + with requests_mock.Mocker() as mock: + mock_api_calls(mock, "bpipe", "0.9.11") + # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) + module_create.create() return root_dir From da53c1a976289a676f2f1ecbf77b7bde2c662005 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 11:13:27 +0200 Subject: [PATCH 348/854] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a25204b21..c8da17542f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ - The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). - Only warn when checking that the pipeline directory contains a `main.nf` and a `nextflow.config` file if the pipeline is not an nf-core pipeline [#1964](https://github.com/nf-core/tools/pull/1964) - Add file `versions.yml` when generating `test.yml` with `nf-core modules create-test-yml` but don't check for md5sum [#1963](https://github.com/nf-core/tools/pull/1963) -- Mock biocontainers and anaconda api calls in modules and subworkflows tests [#1966](https://github.com/nf-core/tools/pull/1966) +- Mock biocontainers and anaconda api calls in modules and subworkflows tests [#1967](https://github.com/nf-core/tools/pull/1967) ### Modules From 4d675bd1290bd46831a9b8a3237dcafb39a400ce Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 12:32:34 +0200 Subject: [PATCH 349/854] add suggestion from @mashehu review --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 8f12a2cc7e..483f817f20 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -82,6 +82,6 @@ def mock_api_calls(mock, module, version): "files": [{"version": version}], "license": "", } - biocontainers_mock = {"status_code": 200, "images": [{"image_type": "Docker", "image_name": ""}]} + biocontainers_mock = {"status_code": 200, "images": [{"image_type": "Docker", "image_name": f"{module}-{version}"}]} mock.register_uri("GET", anaconda_api_url, json=anaconda_mock) mock.register_uri("GET", biocontainers_api_url, json=biocontainers_mock) From 4d12a30f19f6330b775d61c1a9bafcf7be4bb04b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 12:53:22 +0200 Subject: [PATCH 350/854] add get_local_components() --- nf_core/components/components_command.py | 9 +++++++++ nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/modules_command.py | 7 ------- nf_core/subworkflows/subworkflows_command.py | 7 ------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index fad48b6322..83f59e5407 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -36,6 +36,15 @@ def __init__(self, component_type, dir, remote_url=None, branch=None, no_pull=Fa except LookupError as e: raise UserWarning(e) + def get_local_components(self): + """ + Get the local modules/subworkflows in a pipeline + """ + local_component_dir = Path(self.dir, self.component_type, "local") + return [ + str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" + ] + def get_modules_clone_modules(self): """ Get the modules repository available in a clone of nf-core/modules diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 034e7b5f4b..2b917a6ea5 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -96,7 +96,7 @@ def __init__( local_module_dir = Path(self.dir, "modules", "local") self.all_local_modules = [ NFCoreModule(m, None, local_module_dir / m, self.repo_type, Path(self.dir), nf_core_module=False) - for m in self.get_local_modules() + for m in self.get_local_components() ] else: diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index d47d050c38..1a17299036 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -15,13 +15,6 @@ class ModuleCommand(ComponentCommand): def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): super().__init__("modules", dir, remote_url, branch, no_pull, hide_progress) - def get_local_modules(self): - """ - Get the local modules in a pipeline - """ - local_module_dir = Path(self.dir, "modules", "local") - return [str(path.relative_to(local_module_dir)) for path in local_module_dir.iterdir() if path.suffix == ".nf"] - def check_patch_paths(self, patch_path, module_name): """ Check that paths in patch files are updated to the new modules path diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index d3b730d5a0..c26a0fefd2 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -29,13 +29,6 @@ def get_subworkflows_clone_subworkflows(self): if "main.nf" in files ] - def get_local_subworkflows(self): - """ - Get the local subworkflows in a pipeline - """ - local_subwf_dir = Path(self.dir, "subworkflows", "local") - return [str(path.relative_to(local_subwf_dir)) for path in local_subwf_dir.iterdir() if path.suffix == ".nf"] - def subworkflows_from_repo(self, install_dir): """ Gets the subworkflows installed from a certain repository From 379e0109ac297ccd639ba5db456450057a83d73d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 13:35:22 +0200 Subject: [PATCH 351/854] Unify functions get_components_clone_modules, component_exists, get_component_dir, install_component, get_avail_components --- nf_core/components/components_command.py | 23 ++-- nf_core/modules/info.py | 2 +- nf_core/modules/install.py | 6 +- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/list.py | 6 +- nf_core/modules/module_test.py | 2 +- nf_core/modules/modules_repo.py | 138 +++++-------------- nf_core/modules/test_yml_builder.py | 2 +- nf_core/modules/update.py | 4 +- nf_core/subworkflows/install.py | 6 +- nf_core/subworkflows/subworkflows_command.py | 13 -- nf_core/subworkflows/subworkflows_test.py | 2 +- nf_core/subworkflows/test_yml_builder.py | 5 +- 13 files changed, 73 insertions(+), 138 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 83f59e5407..a2d63d7f76 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -28,6 +28,8 @@ def __init__(self, component_type, dir, remote_url=None, branch=None, no_pull=Fa self.hide_progress = hide_progress self.default_modules_path = Path("modules", "nf-core") self.default_tests_path = Path("tests", "modules", "nf-core") + self.default_subworkflows_path = Path("subworkflows", "nf-core") + self.default_subworkflows_tests_path = Path("tests", "subworkflows", "nf-core") try: if self.dir: self.dir, self.repo_type = get_repo_type(self.dir) @@ -45,14 +47,17 @@ def get_local_components(self): str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] - def get_modules_clone_modules(self): + def get_components_clone_modules(self): """ - Get the modules repository available in a clone of nf-core/modules + Get the modules/subworkflows repository available in a clone of nf-core/modules """ - module_base_path = Path(self.dir, self.default_modules_path) + if self.component_type == "modules": + component_base_path = Path(self.dir, self.default_modules_path) + elif self.component_type == "subworkflows": + component_base_path = Path(self.dir, self.default_subworkflows_path) return [ - str(Path(dir).relative_to(module_base_path)) - for dir, _, files in os.walk(module_base_path) + str(Path(dir).relative_to(component_base_path)) + for dir, _, files in os.walk(component_base_path) if "main.nf" in files ] @@ -97,15 +102,15 @@ def clear_module_dir(self, module_name, module_dir): def modules_from_repo(self, install_dir): """ - Gets the modules installed from a certain repository + Gets the modules/subworkflows installed from a certain repository Args: - install_dir (str): The name of the directory where modules are installed + install_dir (str): The name of the directory where modules/subworkflows are installed Returns: - [str]: The names of the modules + [str]: The names of the modules/subworkflows """ - repo_dir = Path(self.dir, "modules", install_dir) + repo_dir = Path(self.dir, self.component_type, install_dir) if not repo_dir.exists(): raise LookupError(f"Nothing installed from {install_dir} in pipeline") diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index e5b45931c7..92fc8b87bd 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -94,7 +94,7 @@ def init_mod_name(self, module): ).unsafe_ask() if self.local: if self.repo_type == "modules": - modules = self.get_modules_clone_modules() + modules = self.get_components_clone_modules() else: modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) modules = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 90295b615a..d6f2bf1b66 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -57,17 +57,17 @@ def install(self, module, silent=False): if module is None: module = questionary.autocomplete( "Tool name:", - choices=self.modules_repo.get_avail_modules(), + choices=self.modules_repo.get_avail_components(self.component_type), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() # Check that the supplied name is an available module - if module and module not in self.modules_repo.get_avail_modules(): + if module and module not in self.modules_repo.get_avail_components(self.component_type): log.error(f"Module '{module}' not found in list of available modules.") log.info("Use the command 'nf-core modules list' to view available software") return False - if not self.modules_repo.module_exists(module): + if not self.modules_repo.component_exists(module, self.component_type): warn_msg = ( f"Module '{module}' not found in remote '{self.modules_repo.remote_url}' ({self.modules_repo.branch})" ) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 2b917a6ea5..9cd000acdd 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -105,7 +105,7 @@ def __init__( module_dir = Path(self.dir, self.default_modules_path) self.all_remote_modules = [ NFCoreModule(m, None, module_dir / m, self.repo_type, Path(self.dir)) - for m in self.get_modules_clone_modules() + for m in self.get_components_clone_modules() ] self.all_local_modules = [] if not self.all_remote_modules: diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index c1fe890a7e..a7725a70cc 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -46,7 +46,11 @@ def pattern_msg(keywords): if self.remote: # Filter the modules by keywords - modules = [mod for mod in self.modules_repo.get_avail_modules() if all(k in mod for k in keywords)] + modules = [ + mod + for mod in self.modules_repo.get_avail_components(self.component_type) + if all(k in mod for k in keywords) + ] # Nothing found if len(modules) == 0: diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 6448d741b7..10b83ad853 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -81,7 +81,7 @@ def _check_inputs(self): # Retrieving installed modules if self.repo_type == "modules": - installed_modules = self.get_modules_clone_modules() + installed_modules = self.get_components_clone_modules() else: modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 85e4a5b573..5621a02249 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -278,92 +278,41 @@ def checkout(self, commit): """ self.repo.git.checkout(commit) - def module_exists(self, module_name, checkout=True): + def component_exists(self, component_name, component_type, checkout=True): """ - Check if a module exists in the branch of the repo + Check if a module/subworkflow exists in the branch of the repo Args: - module_name (str): The name of the module - - Returns: - (bool): Whether the module exists in this branch of the repository - """ - return module_name in self.get_avail_modules(checkout=checkout) - - def subworkflow_exists(self, subworkflow_name, checkout=True): - """ - Check if a subworkflow exists in the branch of the repo - - Args: - subworkflow_name (str): The name of the subworkflow - - Returns: - (bool): Whether the subworkflow exists in this branch of the repository - """ - return subworkflow_name in self.get_avail_subworkflows(checkout=checkout) - - def get_module_dir(self, module_name): - """ - Returns the file path of a module directory in the repo. - Does not verify that the path exists. - Args: - module_name (str): The name of the module + component_name (str): The name of the module/subworkflow Returns: - module_path (str): The path of the module in the local copy of the repository + (bool): Whether the module/subworkflow exists in this branch of the repository """ - return os.path.join(self.modules_dir, module_name) + return component_name in self.get_avail_components(component_type, checkout=checkout) - def get_subworkflow_dir(self, subworkflow_name): + def get_component_dir(self, component_name, component_type): """ - Returns the file path of a subworkflow directory in the repo. + Returns the file path of a module/subworkflow directory in the repo. Does not verify that the path exists. Args: - subworkflow_name (str): The name of the subworkflow - - Returns: - subworkflow_path (str): The path of the subworkflow in the local copy of the repository - """ - return os.path.join(self.subworkflows_dir, subworkflow_name) - - def install_module(self, module_name, install_dir, commit): - """ - Install the module files into a pipeline at the given commit - - Args: - module_name (str): The name of the module - install_dir (str): The path where the module should be installed - commit (str): The git SHA for the version of the module to be installed + component_name (str): The name of the module/subworkflow Returns: - (bool): Whether the operation was successful or not + component_path (str): The path of the module/subworkflow in the local copy of the repository """ - # Check out the repository at the requested ref - try: - self.checkout(commit) - except git.GitCommandError: - return False - - # Check if the module exists in the branch - if not self.module_exists(module_name, checkout=False): - log.error(f"The requested module does not exists in the branch '{self.branch}' of {self.remote_url}'") - return False - - # Copy the files from the repo to the install folder - shutil.copytree(self.get_module_dir(module_name), Path(install_dir, module_name)) - - # Switch back to the tip of the branch - self.checkout_branch() - return True + if component_type == "modules": + return os.path.join(self.modules_dir, component_name) + elif component_type == "subworkflows": + return os.path.join(self.subworkflows_dir, component_name) - def install_subworkflow(self, subworkflow_name, install_dir, commit): + def install_component(self, component_name, install_dir, commit, component_type): """ - Install the subworkflow files into a pipeline at the given commit + Install the module/subworkflow files into a pipeline at the given commit Args: - subworkflow_name (str): The name of the subworkflow - install_dir (str): The path where the subworkflow should be installed - commit (str): The git SHA for the version of the subworkflow to be installed + component_name (str): The name of the module/subworkflow + install_dir (str): The path where the module/subworkflow should be installed + commit (str): The git SHA for the version of the module/subworkflow to be installed Returns: (bool): Whether the operation was successful or not @@ -374,13 +323,15 @@ def install_subworkflow(self, subworkflow_name, install_dir, commit): except git.GitCommandError: return False - # Check if the subworkflow exists in the branch - if not self.subworkflow_exists(subworkflow_name, checkout=False): - log.error(f"The requested subworkflow does not exists in the branch '{self.branch}' of {self.remote_url}'") + # Check if the module/subworkflow exists in the branch + if not self.component_exists(component_name, component_type, checkout=False): + log.error( + f"The requested {component_type[:-1]} does not exists in the branch '{self.branch}' of {self.remote_url}'" + ) return False # Copy the files from the repo to the install folder - shutil.copytree(self.get_subworkflow_dir(subworkflow_name), Path(install_dir, subworkflow_name)) + shutil.copytree(self.get_component_dir(component_name, component_type), Path(install_dir, component_name)) # Switch back to the tip of the branch self.checkout_branch() @@ -401,7 +352,7 @@ def module_files_identical(self, module_name, base_path, commit): else: self.checkout(commit) module_files = ["main.nf", "meta.yml"] - module_dir = self.get_module_dir(module_name) + module_dir = self.get_component_dir(module_name, "modules") files_identical = {file: True for file in module_files} for file in module_files: try: @@ -504,41 +455,28 @@ def get_commit_info(self, sha): return message, date raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'") - def get_avail_modules(self, checkout=True): - """ - Gets the names of the modules in the repository. They are detected by - checking which directories have a 'main.nf' file - - Returns: - ([ str ]): The module names - """ - if checkout: - self.checkout_branch() - # Module directories are characterized by having a 'main.nf' file - avail_module_names = [ - os.path.relpath(dirpath, start=self.modules_dir) - for dirpath, _, file_names in os.walk(self.modules_dir) - if "main.nf" in file_names - ] - return avail_module_names - - def get_avail_subworkflows(self, checkout=True): + def get_avail_components(self, component_type, checkout=True): """ - Gets the names of the subworkflows in the repository. They are detected by + Gets the names of the modules/subworkflows in the repository. They are detected by checking which directories have a 'main.nf' file Returns: - ([ str ]): The subworkflow names + ([ str ]): The module/subworkflow names """ if checkout: self.checkout_branch() - # Module directories are characterized by having a 'main.nf' file - avail_subworkflow_names = [ - os.path.relpath(dirpath, start=self.subworkflows_dir) - for dirpath, _, file_names in os.walk(self.subworkflows_dir) + # Get directory + if component_type == "modules": + directory = self.modules_dir + elif component_type == "subworkflows": + directory = self.subworkflows_dir + # Module/Subworkflow directories are characterized by having a 'main.nf' file + avail_component_names = [ + os.path.relpath(dirpath, start=directory) + for dirpath, _, file_names in os.walk(directory) if "main.nf" in file_names ] - return avail_subworkflow_names + return avail_component_names def get_meta_yml(self, module_name): """ diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index c6ba65f161..7f598b1b92 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -77,7 +77,7 @@ def check_inputs(self): modules_repo = ModulesRepo() self.module_name = questionary.autocomplete( "Tool name:", - choices=modules_repo.get_avail_modules(), + choices=modules_repo.get_avail_components(self.component_type), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() self.module_dir = os.path.join(self.default_modules_path, *self.module_name.split("/")) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index f40b3743f9..645abaca04 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -287,7 +287,7 @@ def get_single_module_info(self, module): raise LookupError(f"Module '{module}' is not installed in pipeline and could therefore not be updated") # Check that the supplied name is an available module - if module and module not in self.modules_repo.get_avail_modules(): + if module and module not in self.modules_repo.get_avail_components(self.component_type): raise LookupError( f"Module '{module}' not found in list of available modules." f"Use the command 'nf-core modules list remote' to view available software" @@ -518,7 +518,7 @@ def get_all_modules_info(self, branch=None): i = 0 while i < len(modules_info): repo, module, sha = modules_info[i] - if not repo.module_exists(module): + if not repo.component_exists(module, self.component_type): log.warning(f"Module '{module}' does not exist in '{repo.remote_url}'. Skipping...") modules_info.pop(i) elif sha is not None and not repo.sha_exists_on_branch(sha): diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index d5584b680d..24df30dbe9 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -57,17 +57,17 @@ def install(self, subworkflow, silent=False): if subworkflow is None: subworkflow = questionary.autocomplete( "Subworkflow name:", - choices=self.modules_repo.get_avail_subworkflows(), + choices=self.modules_repo.get_avail_components(self.component_type), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() # Check that the supplied name is an available subworkflow - if subworkflow and subworkflow not in self.modules_repo.get_avail_subworkflows(): + if subworkflow and subworkflow not in self.modules_repo.get_avail_components(self.component_type): log.error(f"Subworkflow '{subworkflow}' not found in list of available subworkflows.") log.info("Use the command 'nf-core subworkflows list' to view available software") return False - if not self.modules_repo.subworkflow_exists(subworkflow): + if not self.modules_repo.component_exists(subworkflow, self.component_type): warn_msg = f"Subworkflow '{subworkflow}' not found in remote '{self.modules_repo.remote_url}' ({self.modules_repo.branch})" log.warning(warn_msg) return False diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index c26a0fefd2..48e62daf57 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -14,21 +14,8 @@ class SubworkflowCommand(ComponentCommand): """ def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): - self.default_subworkflows_path = Path("subworkflows", "nf-core") - self.default_subworkflows_tests_path = Path("tests", "subworkflows", "nf-core") super().__init__("subworkflows", dir, remote_url, branch, no_pull, hide_progress) - def get_subworkflows_clone_subworkflows(self): - """ - Get the subworkflows repository available in a clone of nf-core/subworkflows - """ - subworkflow_base_path = Path(self.dir, self.default_subworkflows_path) - return [ - str(Path(dir).relative_to(subworkflow_base_path)) - for dir, _, files in os.walk(subworkflow_base_path) - if "main.nf" in files - ] - def subworkflows_from_repo(self, install_dir): """ Gets the subworkflows installed from a certain repository diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 2e0f614a99..d50deda94e 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -82,7 +82,7 @@ def _check_inputs(self): # Retrieving installed modules if self.repo_type == "modules": - installed_subwf = self.get_subworkflows_clone_subworkflows() + installed_subwf = self.get_components_clone_modules() else: modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 561d2d4033..c71295cfe3 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -27,11 +27,12 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo +from tools.nf_core.subworkflows.subworkflows_command import SubworkflowCommand log = logging.getLogger(__name__) -class SubworkflowTestYmlBuilder(object): +class SubworkflowTestYmlBuilder(SubworkflowCommand): def __init__( self, subworkflow=None, @@ -76,7 +77,7 @@ def check_inputs(self): if self.subworkflow is None: self.subworkflow = questionary.autocomplete( "Subworkflow name:", - choices=self.modules_repo.get_avail_subworkflows(), + choices=self.modules_repo.get_avail_components(self.component_type), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() self.subworkflow_dir = os.path.join("subworkflows", self.modules_repo.repo_path, self.subworkflow) From 46daad1615276ebb91ed33858c55e79b62e7c32f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 13:48:17 +0200 Subject: [PATCH 352/854] add install_component_files and resolve conflicts from upstream --- nf_core/components/components_command.py | 16 +++++----- nf_core/modules/info.py | 4 +-- nf_core/modules/install.py | 2 +- nf_core/modules/patch.py | 2 +- nf_core/modules/update.py | 2 +- nf_core/subworkflows/subworkflows_command.py | 33 -------------------- tests/modules/patch.py | 6 ++-- 7 files changed, 17 insertions(+), 48 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index a2d63d7f76..1822d2a290 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -71,7 +71,9 @@ def has_valid_directory(self): main_nf = os.path.join(self.dir, "main.nf") nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + if Path(self.dir).resolve().parts[-1].startswith("nf-core"): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") return True def has_modules_file(self): @@ -118,20 +120,20 @@ def modules_from_repo(self, install_dir): str(Path(dir_path).relative_to(repo_dir)) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files ] - def install_module_files(self, module_name, module_version, modules_repo, install_dir): + def install_component_files(self, component_name, component_version, modules_repo, install_dir): """ - Installs a module into the given directory + Installs a module/subworkflow into the given directory Args: - module_name (str): The name of the module - module_version (str): Git SHA for the version of the module to be installed + component_name (str): The name of the module/subworkflow + component_version (str): Git SHA for the version of the module/subworkflow to be installed modules_repo (ModulesRepo): A correctly configured ModulesRepo object - install_dir (str): The path to where the module should be installed (should be the 'modules/' dir of the pipeline) + install_dir (str): The path to where the module/subworkflow should be installed (should be the 'modules/' or 'subworkflows/' dir of the pipeline) Returns: (bool): Whether the operation was successful of not """ - return modules_repo.install_module(module_name, install_dir, module_version) + return modules_repo.install_component(component_name, install_dir, component_version, self.component_type) def load_lint_config(self): """Parse a pipeline lint config file. diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 92fc8b87bd..a900d4b501 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -101,7 +101,7 @@ def init_mod_name(self, module): if modules is None: raise UserWarning(f"No modules installed from '{self.modules_repo.remote_url}'") else: - modules = self.modules_repo.get_avail_modules() + modules = self.modules_repo.get_avail_components(self.component_type) module = questionary.autocomplete( "Please select a module", choices=modules, style=nf_core.utils.nfcore_question_style ).unsafe_ask() @@ -179,7 +179,7 @@ def get_remote_yaml(self): dict or bool: Parsed meta.yml found, False otherwise """ # Check if our requested module is there - if self.module not in self.modules_repo.get_avail_modules(): + if self.module not in self.modules_repo.get_avail_components(self.component_type): return False file_contents = self.modules_repo.get_meta_yml(self.module) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index d6f2bf1b66..6831402e54 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -139,7 +139,7 @@ def install(self, module, silent=False): log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}") # Download module files - if not self.install_module_files(module, version, self.modules_repo, install_folder): + if not self.install_component_files(module, version, self.modules_repo, install_folder): return False if not silent: diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 565a193729..19b3d6a407 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -86,7 +86,7 @@ def patch(self, module=None): # Create a temporary directory for storing the unchanged version of the module install_dir = tempfile.mkdtemp() module_install_dir = Path(install_dir, module) - if not self.install_module_files(module, module_version, self.modules_repo, install_dir): + if not self.install_component_files(module, module_version, self.modules_repo, install_dir): raise UserWarning( f"Failed to install files of module '{module}' from remote ({self.modules_repo.remote_url})." ) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 645abaca04..46c9c2833c 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -166,7 +166,7 @@ def update(self, module=None): continue # Download module files - if not self.install_module_files(module, version, modules_repo, install_tmp_dir): + if not self.install_component_files(module, version, modules_repo, install_tmp_dir): exit_value = False continue diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index 48e62daf57..ba13579943 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -16,39 +16,6 @@ class SubworkflowCommand(ComponentCommand): def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): super().__init__("subworkflows", dir, remote_url, branch, no_pull, hide_progress) - def subworkflows_from_repo(self, install_dir): - """ - Gets the subworkflows installed from a certain repository - - Args: - install_dir (str): The name of the directory where subworkflows are installed - - Returns: - [str]: The names of the subworkflows - """ - repo_dir = Path(self.dir, "subworkflows", install_dir) - if not repo_dir.exists(): - raise LookupError(f"Nothing installed from {install_dir} in pipeline") - - return [ - str(Path(dir_path).relative_to(repo_dir)) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files - ] - - def install_subworkflow_files(self, subworkflow_name, subworkflow_version, modules_repo, install_dir): - """ - Installs a subworkflow into the given directory - - Args: - subworkflow_name (str): The name of the subworkflow - subworkflow_version (str): Git SHA for the version of the subworkflow to be installed - modules_repo (ModulesRepo): A correctly configured ModulesRepo object - install_dir (str): The path to where the subworkflow should be installed (should be the 'subworkflow/' dir of the pipeline) - - Returns: - (bool): Whether the operation was successful of not - """ - return modules_repo.install_subworkflow(subworkflow_name, install_dir, subworkflow_version) - def clear_subworkflow_dir(self, subworkflow_name, subworkflow_dir): """Removes all files in the subworkflow directory""" try: diff --git a/tests/modules/patch.py b/tests/modules/patch.py index fe736c447a..0f7a7c9101 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -136,7 +136,7 @@ def test_create_patch_try_apply_successful(self): ) # Install the new files install_dir = Path(tempfile.mkdtemp()) - update_obj.install_module_files(BISMARK_ALIGN, SUCCEED_SHA, update_obj.modules_repo, install_dir) + update_obj.install_component_files(BISMARK_ALIGN, SUCCEED_SHA, update_obj.modules_repo, install_dir) # Try applying the patch module_install_dir = install_dir / BISMARK_ALIGN @@ -202,7 +202,7 @@ def test_create_patch_try_apply_failed(self): ) # Install the new files install_dir = Path(tempfile.mkdtemp()) - update_obj.install_module_files(BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, install_dir) + update_obj.install_component_files(BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, install_dir) # Try applying the patch module_install_dir = install_dir / BISMARK_ALIGN @@ -306,7 +306,7 @@ def test_create_patch_update_fail(self): # Check that the installed files have not been affected by the attempted patch temp_dir = Path(tempfile.mkdtemp()) - nf_core.modules.modules_command.ModuleCommand(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH).install_module_files( + nf_core.modules.modules_command.ModuleCommand(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH).install_component_files( BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir ) From 5d4642608756a715e918d613bb849f78af29fd52 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 13:58:57 +0200 Subject: [PATCH 353/854] fix failing tests because of missing functions --- nf_core/modules/modules_json.py | 2 +- nf_core/subworkflows/test_yml_builder.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 3515bde2a4..a095049e65 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -446,7 +446,7 @@ def reinstall_repo(self, install_dir, remote_url, module_entries): log.error(e) failed_to_install.extend(modules) for module, sha in modules: - if not modules_repo.install_module(module, self.modules_dir / install_dir, sha): + if not modules_repo.install_component(module, self.modules_dir / install_dir, sha, "modules"): log.warning( f"Could not install module '{Path(self.modules_dir, install_dir, module)}' - removing from modules.json" ) diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index c71295cfe3..59729ba41d 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -27,7 +27,7 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo -from tools.nf_core.subworkflows.subworkflows_command import SubworkflowCommand +from nf_core.subworkflows.subworkflows_command import SubworkflowCommand log = logging.getLogger(__name__) @@ -42,7 +42,7 @@ def __init__( force_overwrite=False, no_prompts=False, ): - # super().__init__(directory) + super().__init__(directory) self.dir = directory self.subworkflow = subworkflow self.run_tests = run_tests From 6f5a9b63c7f9640abcfd41b77c7291be147cbb05 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 14:25:49 +0200 Subject: [PATCH 354/854] unify clear_component_dir --- nf_core/components/components_command.py | 31 ++++++++++---------- nf_core/modules/install.py | 2 +- nf_core/modules/remove.py | 4 +-- nf_core/modules/update.py | 2 +- nf_core/subworkflows/install.py | 2 +- nf_core/subworkflows/subworkflows_command.py | 10 ------- 6 files changed, 21 insertions(+), 30 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 1822d2a290..dd1f57e1d7 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -83,26 +83,27 @@ def has_modules_file(self): log.info("Creating missing 'module.json' file.") ModulesJson(self.dir).create() - def clear_module_dir(self, module_name, module_dir): - """Removes all files in the module directory""" + def clear_component_dir(self, component_name, component_dir): + """Removes all files in the module/subworkflow directory""" try: - shutil.rmtree(module_dir) - # Try cleaning up empty parent if tool/subtool and tool/ is empty - if module_name.count("/") > 0: - parent_dir = os.path.dirname(module_dir) - try: - os.rmdir(parent_dir) - except OSError: - log.debug(f"Parent directory not empty: '{parent_dir}'") - else: - log.debug(f"Deleted orphan tool directory: '{parent_dir}'") - log.debug(f"Successfully removed {module_name} module") + shutil.rmtree(component_dir) + if self.component_type == "modules": + # Try cleaning up empty parent if tool/subtool and tool/ is empty + if component_name.count("/") > 0: + parent_dir = os.path.dirname(component_dir) + try: + os.rmdir(parent_dir) + except OSError: + log.debug(f"Parent directory not empty: '{parent_dir}'") + else: + log.debug(f"Deleted orphan tool directory: '{parent_dir}'") + log.debug(f"Successfully removed {component_name} {self.component_type[:-1]}") return True except OSError as e: - log.error(f"Could not remove module: {e}") + log.error(f"Could not remove {self.component_type[:-1]}: {e}") return False - def modules_from_repo(self, install_dir): + def components_from_repo(self, install_dir): """ Gets the modules/subworkflows installed from a certain repository diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 6831402e54..a787ac8077 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -123,7 +123,7 @@ def install(self, module, silent=False): if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") - self.clear_module_dir(module, module_dir) + self.clear_component_dir(module, module_dir) for repo_url, repo_content in modules_json.modules_json["repos"].items(): for dir, dir_modules in repo_content["modules"].items(): for name, _ in dir_modules.items(): diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 9f54efca37..cfe0b3c557 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -33,7 +33,7 @@ def remove(self, module): if module is None: module = questionary.autocomplete( "Tool name:", - choices=self.modules_from_repo(repo_dir), + choices=self.components_from_repo(repo_dir), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -59,4 +59,4 @@ def remove(self, module): modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) # Remove the module - return self.clear_module_dir(module_name=module, module_dir=module_dir) + return self.clear_component_dir(module_name=module, module_dir=module_dir) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 46c9c2833c..ce39b4b765 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -580,7 +580,7 @@ def move_files_from_tmp_dir(self, module, install_folder, repo_path, new_version pipeline_path = os.path.join(self.dir, "modules", repo_path, module) log.debug(f"Removing old version of module '{module}'") - self.clear_module_dir(module, pipeline_path) + self.clear_component_dir(module, pipeline_path) os.makedirs(pipeline_path) for file in files: diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 24df30dbe9..f0959e92d9 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -121,7 +121,7 @@ def install(self, subworkflow, silent=False): if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") - self.clear_subworkflow_dir(subworkflow, subworkflow_dir) + self.clear_component_dir(subworkflow, subworkflow_dir) for repo_url, repo_content in modules_json.modules_json["repos"].items(): for dir, dir_subworkflow in repo_content["subworkflows"].items(): for name, _ in dir_subworkflow.items(): diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index ba13579943..e6d48d2a3f 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -15,13 +15,3 @@ class SubworkflowCommand(ComponentCommand): def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): super().__init__("subworkflows", dir, remote_url, branch, no_pull, hide_progress) - - def clear_subworkflow_dir(self, subworkflow_name, subworkflow_dir): - """Removes all files in the subworkflow directory""" - try: - shutil.rmtree(subworkflow_dir) - log.debug(f"Successfully removed {subworkflow_name} subworkflow") - return True - except OSError as e: - log.error(f"Could not remove subworkflow: {e}") - return False From acd6d9bb8a51e87e26aef61ef3daf2fa586f26e9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 14:27:42 +0200 Subject: [PATCH 355/854] move check_modules_structure to modules_command --- nf_core/components/components_command.py | 37 ------------------------ nf_core/modules/modules_command.py | 37 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index dd1f57e1d7..231e69261a 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -157,40 +157,3 @@ def load_lint_config(self): self.lint_config = yaml.safe_load(fh) except FileNotFoundError: log.debug(f"No lint config file found: {config_fn}") - - def check_modules_structure(self): - """ - Check that the structure of the modules directory in a pipeline is the correct one: - modules/nf-core/TOOL/SUBTOOL - - Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: - modules/nf-core/modules/TOOL/SUBTOOL - """ - if self.repo_type == "pipeline": - wrong_location_modules = [] - for directory, _, files in os.walk(Path(self.dir, "modules")): - if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir, "modules")) - parts = module_path.parts - # Check that there are modules installed directly under the 'modules' directory - if parts[1] == "modules": - wrong_location_modules.append(module_path) - # If there are modules installed in the wrong location - if len(wrong_location_modules) > 0: - log.info("The modules folder structure is outdated. Reinstalling modules.") - # Remove the local copy of the modules repository - log.info(f"Updating '{self.modules_repo.local_repo_dir}'") - self.modules_repo.setup_local_repo( - self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress - ) - # Move wrong modules to the right directory - for module in wrong_location_modules: - modules_dir = Path("modules").resolve() - correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) - wrong_dir = Path(modules_dir, module) - shutil.move(wrong_dir, correct_dir) - log.info(f"Moved {wrong_dir} to {correct_dir}.") - shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) - # Regenerate modules.json file - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 1a17299036..083975494e 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -45,3 +45,40 @@ def check_patch_paths(self, patch_path, module_name): self.modules_repo.repo_path ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) modules_json.dump() + + def check_modules_structure(self): + """ + Check that the structure of the modules directory in a pipeline is the correct one: + modules/nf-core/TOOL/SUBTOOL + + Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: + modules/nf-core/modules/TOOL/SUBTOOL + """ + if self.repo_type == "pipeline": + wrong_location_modules = [] + for directory, _, files in os.walk(Path(self.dir, "modules")): + if "main.nf" in files: + module_path = Path(directory).relative_to(Path(self.dir, "modules")) + parts = module_path.parts + # Check that there are modules installed directly under the 'modules' directory + if parts[1] == "modules": + wrong_location_modules.append(module_path) + # If there are modules installed in the wrong location + if len(wrong_location_modules) > 0: + log.info("The modules folder structure is outdated. Reinstalling modules.") + # Remove the local copy of the modules repository + log.info(f"Updating '{self.modules_repo.local_repo_dir}'") + self.modules_repo.setup_local_repo( + self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress + ) + # Move wrong modules to the right directory + for module in wrong_location_modules: + modules_dir = Path("modules").resolve() + correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) + wrong_dir = Path(modules_dir, module) + shutil.move(wrong_dir, correct_dir) + log.info(f"Moved {wrong_dir} to {correct_dir}.") + shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) + # Regenerate modules.json file + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() From 4bcd4a99a18efbc3d4aa089d6d316b6a3ee437b4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 14:32:37 +0200 Subject: [PATCH 356/854] remove check_modules_structure from subworkflows_test --- nf_core/subworkflows/subworkflows_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index d50deda94e..82f2b1441f 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -77,9 +77,6 @@ def run(self): def _check_inputs(self): """Do more complex checks about supplied flags.""" - # Check modules directory structure - self.check_modules_structure() - # Retrieving installed modules if self.repo_type == "modules": installed_subwf = self.get_components_clone_modules() From 2b164ed34c3e2cdbee61f3da3aed55c70d0ff846 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 16:27:29 +0200 Subject: [PATCH 357/854] poc for shared functions in components commands --- nf_core/components/components_create.py | 169 ++++++++++++++++++ nf_core/components/components_utils.py | 109 ------------ nf_core/modules/create.py | 227 ++++++------------------ nf_core/subworkflows/create.py | 130 +++----------- 4 files changed, 244 insertions(+), 391 deletions(-) create mode 100644 nf_core/components/components_create.py diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py new file mode 100644 index 0000000000..d26e22437f --- /dev/null +++ b/nf_core/components/components_create.py @@ -0,0 +1,169 @@ +import glob +import json +import logging +import os +import re + +import jinja2 +import rich + +import nf_core.utils + +log = logging.getLogger(__name__) + + +def render_template(component_type, object_attrs, file_paths): + """ + Create new module/subworkflow files with Jinja2. + """ + # Run jinja2 for each file in the template folder + env = jinja2.Environment( + loader=jinja2.PackageLoader("nf_core", f"{component_type[:-1]}-template"), keep_trailing_newline=True + ) + for template_fn, dest_fn in file_paths.items(): + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + object_attrs["nf_core_version"] = nf_core.__version__ + rendered_output = j_template.render(object_attrs) + + # Write output to the target file + os.makedirs(os.path.dirname(dest_fn), exist_ok=True) + with open(dest_fn, "w") as fh: + log.debug(f"Writing output to: '{dest_fn}'") + fh.write(rendered_output) + + # Mirror file permissions + template_stat = os.stat( + os.path.join(os.path.dirname(nf_core.__file__), f"{component_type[:-1]}-template", template_fn) + ) + os.chmod(dest_fn, template_stat.st_mode) + + +def collect_name_prompt(name, component_type): + """ + Collect module/subworkflow info via prompt if empty or invalid + """ + # Collect module info via prompt if empty or invalid + if name is None: + name = "" + while name == "" or re.search(r"[^a-z\d/]", name) or name.count("/") > 0: + # Check + auto-fix for invalid chacters + if re.search(r"[^a-z\d/]", name): + if component_type == "modules": + log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") + elif component_type == "subworkflows": + log.warning("Subworkflow name must be lower-case letters only, with no punctuation") + name_clean = re.sub(r"[^a-z\d/]", "", name.lower()) + if rich.prompt.Confirm.ask(f"[violet]Change '{name}' to '{name_clean}'?"): + name = name_clean + else: + name = "" + + if component_type == "modules": + # Split into tool and subtool + if name.count("/") > 1: + log.warning("Tool/subtool can have maximum one '/' character") + name = "" + elif name.count("/") == 1: + name, subname = name.split("/") + else: + subname = None # Reset edge case: entered '/subtool' as name and gone round loop again + + # Prompt for new entry if we reset + if name == "": + if component_type == "modules": + name = rich.prompt.Prompt.ask("[violet]Name of tool/subtool").strip() + elif component_type == "subworkflows": + name = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() + + if component_type == "modules": + return name, subname + elif component_type == "subworkflows": + return name + + +def get_component_dirs(component_type, repo_type, directory, name, supername, subname, new_dir, force_overwrite): + """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist + + Returns dict: keys are relative paths to template files, vals are target paths. + """ + file_paths = {} + if repo_type == "pipeline": + local_component_dir = os.path.join(directory, component_type, "local") + # Check whether component file already exists + component_file = os.path.join(local_component_dir, f"{name}.nf") + if os.path.exists(component_file) and not force_overwrite: + raise UserWarning( + f"{component_type[:-1]} file exists already: '{component_file}'. Use '--force' to overwrite" + ) + + if component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + if subname and os.path.exists(os.path.join(local_component_dir, f"{supername}.nf")): + raise UserWarning(f"Module '{supername}' exists already, cannot make subtool '{name}'") + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{local_component_dir}/{supername}_*.nf") + if not subname and tool_glob: + raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'") + + # Set file paths + file_paths[os.path.join(component_type, "main.nf")] = component_file + + if repo_type == "modules": + software_dir = os.path.join(directory, component_type, "nf-core", new_dir) + test_dir = os.path.join(directory, "tests", component_type, "nf-core", new_dir) + + # Check if module/subworkflow directories exist already + if os.path.exists(software_dir) and not force_overwrite: + raise UserWarning(f"{component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite") + if os.path.exists(test_dir) and not force_overwrite: + raise UserWarning(f"{component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite") + + if component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = os.path.join(directory, component_type, "nf-core", supername, "main.nf") + parent_tool_test_nf = os.path.join(directory, component_type, "nf-core", supername, "main.nf") + if subname and os.path.exists(parent_tool_main_nf): + raise UserWarning(f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{name}'") + if subname and os.path.exists(parent_tool_test_nf): + raise UserWarning(f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{name}'") + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{os.path.join(directory, component_type, 'nf-core', supername)}/*/main.nf") + if not subname and tool_glob: + raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'") + + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths[os.path.join(component_type, "main.nf")] = os.path.join(software_dir, "main.nf") + file_paths[os.path.join(component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") + file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") + file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") + file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") + + return file_paths + + +def get_username(author): + """ + Prompt for GitHub username + """ + # Try to guess the current user if `gh` is installed + author_default = None + try: + with open(os.devnull, "w") as devnull: + gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) + author_default = f"@{gh_auth_user['login']}" + except Exception as e: + log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") + + # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex + github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") + while author is None or not github_username_regex.match(author): + if author is not None and not github_username_regex.match(author): + log.warning("Does not look like a valid GitHub username (must start with an '@')!") + author = rich.prompt.Prompt.ask( + f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", + default=author_default, + ) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index fe916c205c..e826cc37c8 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -67,112 +67,3 @@ def get_repo_type(dir, repo_type=None, use_prompt=True): # It was set on the command line, return what we were given return [dir, repo_type] - - -def get_component_dirs(self, component): - """ - Given a directory and a tool/subtool or subworkflow name, set the file paths and check if they already exist - Args: - component (str): Type of component. "modules" or "subworkflows". - - Returns dict: keys are relative paths to template files, vals are target paths. - """ - - file_paths = {} - - if self.repo_type == "pipeline": - local_component_dir = os.path.join(self.directory, component, "local") - if component == "modules": - component_name = self.tool_name - elif component == "subworkflows": - component_name = self.subworkflow_name - - # Check whether component file already exists - component_file = os.path.join(local_component_dir, f"{component_name}.nf") - if os.path.exists(component_file) and not self.force_overwrite: - raise UserWarning(f"{component[:-1]} file exists already: '{component_file}'. Use '--force' to overwrite") - - # If a subtool, check if there is a module called the base tool name already - if self.subtool and os.path.exists(os.path.join(local_modules_dir, f"{self.tool}.nf")): - raise UserWarning(f"Module '{self.tool}' exists already, cannot make subtool '{self.tool_name}'") - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_modules_dir}/{self.tool}_*.nf") - if not self.subtool and tool_glob: - raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'") - - # Set file paths - file_paths[os.path.join("modules", "main.nf")] = module_file - - if self.repo_type == "modules": - software_dir = os.path.join(self.directory, self.default_modules_path, self.tool_dir) - test_dir = os.path.join(self.directory, self.default_tests_path, self.tool_dir) - - # Check if module directories exist already - if os.path.exists(software_dir) and not self.force_overwrite: - raise UserWarning(f"Module directory exists: '{software_dir}'. Use '--force' to overwrite") - - if os.path.exists(test_dir) and not self.force_overwrite: - raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite") - - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = os.path.join(self.directory, self.default_modules_path, self.tool, "main.nf") - parent_tool_test_nf = os.path.join(self.directory, self.default_tests_path, self.tool, "main.nf") - if self.subtool and os.path.exists(parent_tool_main_nf): - raise UserWarning(f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.tool_name}'") - if self.subtool and os.path.exists(parent_tool_test_nf): - raise UserWarning(f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{self.tool_name}'") - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{os.path.join(self.directory, self.default_modules_path, self.tool)}/*/main.nf") - if not self.subtool and tool_glob: - raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'") - - # Set file paths - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join("modules", "main.nf")] = os.path.join(software_dir, "main.nf") - file_paths[os.path.join("modules", "meta.yml")] = os.path.join(software_dir, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") - - return file_paths - - -def get_subworkflow_dirs(self): - """Given a directory and a subworkflow, set the file paths and check if they already exist - - Returns dict: keys are relative paths to template files, vals are target paths. - """ - - file_paths = {} - - if self.repo_type == "pipeline": - local_subworkflow_dir = os.path.join(self.directory, "subworkflows", "local") - - # Check whether subworkflow file already exists - subworkflow_file = os.path.join(local_subworkflow_dir, f"{self.subworkflow_name}.nf") - if os.path.exists(subworkflow_file) and not self.force_overwrite: - raise UserWarning(f"Subworkflow file exists already: '{subworkflow_file}'. Use '--force' to overwrite") - - # Set file paths - file_paths[os.path.join("subworkflows", "main.nf")] = subworkflow_file - - if self.repo_type == "modules": - subworkflow_path = os.path.join(self.directory, "subworkflows", "nf-core", self.subworkflow_dir) - test_dir = os.path.join(self.directory, "tests", "subworkflows", "nf-core", self.subworkflow_dir) - - # Check if module directories exist already - if os.path.exists(subworkflow_path) and not self.force_overwrite: - raise UserWarning(f"Subworkflow directory exists: '{subworkflow_path}'. Use '--force' to overwrite") - - if os.path.exists(test_dir) and not self.force_overwrite: - raise UserWarning(f"Subworkflow test directory exists: '{test_dir}'. Use '--force' to overwrite") - - # Set file paths - file_paths[os.path.join("subworkflows", "main.nf")] = os.path.join(subworkflow_path, "main.nf") - file_paths[os.path.join("subworkflows", "meta.yml")] = os.path.join(subworkflow_path, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") - - return file_paths diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index f57685e7de..2828fd6aea 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -5,20 +5,16 @@ from __future__ import print_function -import glob -import json import logging import os -import re -import subprocess -import jinja2 import questionary import rich import yaml from packaging.version import parse as parse_version import nf_core +import nf_core.components.components_create import nf_core.modules.module_utils import nf_core.utils @@ -106,31 +102,9 @@ def create(self): ) # Collect module info via prompt if empty or invalid - if self.tool is None: - self.tool = "" - while self.tool == "" or re.search(r"[^a-z\d/]", self.tool) or self.tool.count("/") > 0: - - # Check + auto-fix for invalid chacters - if re.search(r"[^a-z\d/]", self.tool): - log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") - tool_clean = re.sub(r"[^a-z\d/]", "", self.tool.lower()) - if rich.prompt.Confirm.ask(f"[violet]Change '{self.tool}' to '{tool_clean}'?"): - self.tool = tool_clean - else: - self.tool = "" - - # Split into tool and subtool - if self.tool.count("/") > 1: - log.warning("Tool/subtool can have maximum one '/' character") - self.tool = "" - elif self.tool.count("/") == 1: - self.tool, self.subtool = self.tool.split("/") - else: - self.subtool = None # Reset edge case: entered '/subtool' as name and gone round loop again - - # Prompt for new entry if we reset - if self.tool == "": - self.tool = rich.prompt.Prompt.ask("[violet]Name of tool/subtool").strip() + self.tool, self.subtool = nf_core.components.components_create.collect_name_prompt( + self.tool, self.component_type + ) # Determine the tool name self.tool_name = self.tool @@ -143,9 +117,58 @@ def create(self): self.tool_name_underscore = self.tool_name.replace("/", "_") # Check existence of directories early for fast-fail - self.file_paths = self.get_module_dirs() + self.file_paths = nf_core.components.components_create.get_component_dirs( + self.component_type, + self.repo_type, + self.directory, + self.tool_name, + self.tool, + self.subtool, + self.tool_dir, + self.force_overwrite, + ) # Try to find a bioconda package for 'tool' + self._get_bioconda_tool() + + # Prompt for GitHub username + nf_core.components.components_create.get_username(self.author) + + self._get_module_structure_components() + + # Create module template with jinja2 + nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) + + if self.repo_type == "modules": + # Add entry to pytest_modules.yml + try: + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: + pytest_modules_yml = yaml.safe_load(fh) + if self.subtool: + pytest_modules_yml[self.tool_name] = [ + f"modules/nf-core/{self.tool}/{self.subtool}/**", + f"tests/modules/nf-core/{self.tool}/{self.subtool}/**", + ] + else: + pytest_modules_yml[self.tool_name] = [ + f"modules/nf-core/{self.tool}/**", + f"tests/modules/nf-core/{self.tool}/**", + ] + pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: + yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) + except FileNotFoundError as e: + raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") + + new_files = list(self.file_paths.values()) + if self.repo_type == "modules": + new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) + log.info("Created / edited following files:\n " + "\n ".join(new_files)) + + def _get_bioconda_tool(self): + """ + Try to find a bioconda package for 'tool' + """ while True: try: if self.tool_conda_name: @@ -199,26 +222,7 @@ def create(self): except (ValueError, LookupError) as e: log.info(f"Could not find a Docker/Singularity container ({e})") - # Prompt for GitHub username - # Try to guess the current user if `gh` is installed - author_default = None - try: - with open(os.devnull, "w") as devnull: - gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) - author_default = f"@{gh_auth_user['login']}" - except Exception as e: - log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") - - # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex - github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") - while self.author is None or not github_username_regex.match(self.author): - if self.author is not None and not github_username_regex.match(self.author): - log.warning("Does not look like a valid GitHub username (must start with an '@')!") - self.author = rich.prompt.Prompt.ask( - f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", - default=author_default, - ) - + def _get_module_structure_components(self): process_label_defaults = ["process_single", "process_low", "process_medium", "process_high", "process_long"] if self.process_label is None: log.info( @@ -245,124 +249,3 @@ def create(self): self.has_meta = rich.prompt.Confirm.ask( "[violet]Will the module require a meta map of sample information?", default=True ) - - # Create module template with jinja - self.render_template() - - if self.repo_type == "modules": - # Add entry to pytest_modules.yml - try: - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: - pytest_modules_yml = yaml.safe_load(fh) - if self.subtool: - pytest_modules_yml[self.tool_name] = [ - f"modules/nf-core/{self.tool}/{self.subtool}/**", - f"tests/modules/nf-core/{self.tool}/{self.subtool}/**", - ] - else: - pytest_modules_yml[self.tool_name] = [ - f"modules/nf-core/{self.tool}/**", - f"tests/modules/nf-core/{self.tool}/**", - ] - pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: - yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError as e: - raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") - - new_files = list(self.file_paths.values()) - if self.repo_type == "modules": - new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) - log.info("Created / edited following files:\n " + "\n ".join(new_files)) - - def render_template(self): - """ - Create new module files with Jinja2. - """ - # Run jinja2 for each file in the template folder - env = jinja2.Environment(loader=jinja2.PackageLoader("nf_core", "module-template"), keep_trailing_newline=True) - for template_fn, dest_fn in self.file_paths.items(): - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - object_attrs = vars(self) - object_attrs["nf_core_version"] = nf_core.__version__ - rendered_output = j_template.render(object_attrs) - - # Write output to the target file - os.makedirs(os.path.dirname(dest_fn), exist_ok=True) - with open(dest_fn, "w") as fh: - log.debug(f"Writing output to: '{dest_fn}'") - fh.write(rendered_output) - - # Mirror file permissions - template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) - os.chmod(dest_fn, template_stat.st_mode) - - def get_module_dirs(self): - """Given a directory and a tool/subtool, set the file paths and check if they already exist - - Returns dict: keys are relative paths to template files, vals are target paths. - """ - - file_paths = {} - - if self.repo_type == "pipeline": - local_modules_dir = os.path.join(self.directory, "modules", "local") - - # Check whether module file already exists - module_file = os.path.join(local_modules_dir, f"{self.tool_name}.nf") - if os.path.exists(module_file) and not self.force_overwrite: - raise UserWarning(f"Module file exists already: '{module_file}'. Use '--force' to overwrite") - - # If a subtool, check if there is a module called the base tool name already - if self.subtool and os.path.exists(os.path.join(local_modules_dir, f"{self.tool}.nf")): - raise UserWarning(f"Module '{self.tool}' exists already, cannot make subtool '{self.tool_name}'") - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_modules_dir}/{self.tool}_*.nf") - if not self.subtool and tool_glob: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'" - ) - - # Set file paths - file_paths[os.path.join("modules", "main.nf")] = module_file - - if self.repo_type == "modules": - software_dir = os.path.join(self.directory, self.default_modules_path, self.tool_dir) - test_dir = os.path.join(self.directory, self.default_tests_path, self.tool_dir) - - # Check if module directories exist already - if os.path.exists(software_dir) and not self.force_overwrite: - raise UserWarning(f"Module directory exists: '{software_dir}'. Use '--force' to overwrite") - - if os.path.exists(test_dir) and not self.force_overwrite: - raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite") - - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = os.path.join(self.directory, self.default_modules_path, self.tool, "main.nf") - parent_tool_test_nf = os.path.join(self.directory, self.default_tests_path, self.tool, "main.nf") - if self.subtool and os.path.exists(parent_tool_main_nf): - raise UserWarning( - f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.tool_name}'" - ) - if self.subtool and os.path.exists(parent_tool_test_nf): - raise UserWarning( - f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{self.tool_name}'" - ) - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{os.path.join(self.directory, self.default_modules_path, self.tool)}/*/main.nf") - if not self.subtool and tool_glob: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'" - ) - - # Set file paths - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join("modules", "main.nf")] = os.path.join(software_dir, "main.nf") - file_paths[os.path.join("modules", "meta.yml")] = os.path.join(software_dir, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") - - return file_paths diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index eb2f8276d9..bc5125f209 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -5,25 +5,21 @@ from __future__ import print_function -import json import logging import os -import re -import subprocess -import jinja2 -import rich import yaml from packaging.version import parse as parse_version import nf_core +import nf_core.components.components_create import nf_core.utils from nf_core.modules.module_utils import get_repo_type log = logging.getLogger(__name__) -class SubworkflowCreate(object): +class SubworkflowCreate(SubworkflowCommand): def __init__( self, directory=".", @@ -32,6 +28,7 @@ def __init__( force=False, repo_type=None, ): + super().__init__(directory) self.directory = directory self.subworkflow = subworkflow self.author = author @@ -77,52 +74,31 @@ def create(self): ) # Collect module info via prompt if empty or invalid - if self.subworkflow is None: - self.subworkflow = "" - while self.subworkflow == "" or re.search(r"[^a-z\d_/]", self.subworkflow) or self.subworkflow.count("/") > 0: - - # Check + auto-fix for invalid characters - if re.search(r"[^a-z\d_/]", self.subworkflow): - log.warning("Subworkflow name must be lower-case letters only, with no punctuation") - subworkflow_clean = re.sub(r"[^a-z\d/]", "", self.subworkflow.lower()) - if rich.prompt.Confirm.ask(f"[violet]Change '{self.subworkflow}' to '{subworkflow_clean}'?"): - self.subworkflow = subworkflow_clean - else: - self.subworkflow = "" - - # Prompt for new entry if we reset - if self.subworkflow == "": - self.subworkflow = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() + self.subworkflow = nf_core.components.components_create.collect_name_prompt( + self.subworkflow, self.component_type + ) # Determine the tool name self.subworkflow_name = self.subworkflow self.subworkflow_dir = self.subworkflow # Check existence of directories early for fast-fail - self.file_paths = self.get_subworkflow_dirs() + self.file_paths = nf_core.components.components_create.get_component_dirs( + self.component_type, + self.repo_type, + self.directory, + self.subworkflow_name, + None, + None, + self.subworkflow_dir, + self.force_overwrite, + ) # Prompt for GitHub username - # Try to guess the current user if `gh` is installed - author_default = None - try: - with open(os.devnull, "w") as devnull: - gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) - author_default = f"@{gh_auth_user['login']}" - except Exception as e: - log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") - - # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex - github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") - while self.author is None or not github_username_regex.match(self.author): - if self.author is not None and not github_username_regex.match(self.author): - log.warning("Does not look like a valid GitHub username (must start with an '@')!") - self.author = rich.prompt.Prompt.ask( - f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", - default=author_default, - ) - - # Create module template with jinja2 - self.render_template() + nf_core.components.components_create.get_username(self.author) + + # Create subworkflow template with jinja2 + nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) if self.repo_type == "modules": # Add entry to pytest_modules.yml @@ -143,69 +119,3 @@ def create(self): if self.repo_type == "modules": new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) log.info("Created / edited following files:\n " + "\n ".join(new_files)) - - def render_template(self): - """ - Create new subworkflow files with Jinja2. - """ - # Run jinja2 for each file in the template folder - env = jinja2.Environment( - loader=jinja2.PackageLoader("nf_core", "subworkflow-template"), keep_trailing_newline=True - ) - for template_fn, dest_fn in self.file_paths.items(): - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - object_attrs = vars(self) - object_attrs["nf_core_version"] = nf_core.__version__ - rendered_output = j_template.render(object_attrs) - - # Write output to the target file - os.makedirs(os.path.dirname(dest_fn), exist_ok=True) - with open(dest_fn, "w") as fh: - log.debug(f"Writing output to: '{dest_fn}'") - fh.write(rendered_output) - - # Mirror file permissions - template_stat = os.stat( - os.path.join(os.path.dirname(nf_core.__file__), "subworkflow-template", template_fn) - ) - os.chmod(dest_fn, template_stat.st_mode) - - def get_subworkflow_dirs(self): - """Given a directory and a subworkflow, set the file paths and check if they already exist - - Returns dict: keys are relative paths to template files, vals are target paths. - """ - - file_paths = {} - - if self.repo_type == "pipeline": - local_subworkflow_dir = os.path.join(self.directory, "subworkflows", "local") - - # Check whether subworkflow file already exists - subworkflow_file = os.path.join(local_subworkflow_dir, f"{self.subworkflow_name}.nf") - if os.path.exists(subworkflow_file) and not self.force_overwrite: - raise UserWarning(f"Subworkflow file exists already: '{subworkflow_file}'. Use '--force' to overwrite") - - # Set file paths - file_paths[os.path.join("subworkflows", "main.nf")] = subworkflow_file - - if self.repo_type == "modules": - subworkflow_path = os.path.join(self.directory, "subworkflows", "nf-core", self.subworkflow_dir) - test_dir = os.path.join(self.directory, "tests", "subworkflows", "nf-core", self.subworkflow_dir) - - # Check if module directories exist already - if os.path.exists(subworkflow_path) and not self.force_overwrite: - raise UserWarning(f"Subworkflow directory exists: '{subworkflow_path}'. Use '--force' to overwrite") - - if os.path.exists(test_dir) and not self.force_overwrite: - raise UserWarning(f"Subworkflow test directory exists: '{test_dir}'. Use '--force' to overwrite") - - # Set file paths - file_paths[os.path.join("subworkflows", "main.nf")] = os.path.join(subworkflow_path, "main.nf") - file_paths[os.path.join("subworkflows", "meta.yml")] = os.path.join(subworkflow_path, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") - - return file_paths From c6cc7a099c910c6d9c1149d9ab3e0c8a13383e00 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Oct 2022 17:41:16 +0200 Subject: [PATCH 358/854] fix bugs --- nf_core/components/components_create.py | 26 +++++++++++++++---------- nf_core/modules/modules_command.py | 2 ++ nf_core/modules/remove.py | 2 +- nf_core/subworkflows/create.py | 2 ++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py index d26e22437f..6b2962498c 100644 --- a/nf_core/components/components_create.py +++ b/nf_core/components/components_create.py @@ -44,11 +44,16 @@ def collect_name_prompt(name, component_type): Collect module/subworkflow info via prompt if empty or invalid """ # Collect module info via prompt if empty or invalid + subname = None + if component_type == "modules": + pattern = r"[^a-z\d/]" + elif component_type == "subworkflows": + pattern = r"[^a-z\d_/]" if name is None: name = "" - while name == "" or re.search(r"[^a-z\d/]", name) or name.count("/") > 0: + while name == "" or re.search(pattern, name) or name.count("/") > 0: # Check + auto-fix for invalid chacters - if re.search(r"[^a-z\d/]", name): + if re.search(pattern, name): if component_type == "modules": log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") elif component_type == "subworkflows": @@ -93,8 +98,9 @@ def get_component_dirs(component_type, repo_type, directory, name, supername, su # Check whether component file already exists component_file = os.path.join(local_component_dir, f"{name}.nf") if os.path.exists(component_file) and not force_overwrite: + print(f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite") raise UserWarning( - f"{component_type[:-1]} file exists already: '{component_file}'. Use '--force' to overwrite" + f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" ) if component_type == "modules": @@ -134,13 +140,13 @@ def get_component_dirs(component_type, repo_type, directory, name, supername, su if not subname and tool_glob: raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'") - # Set file paths - # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join(component_type, "main.nf")] = os.path.join(software_dir, "main.nf") - file_paths[os.path.join(component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths[os.path.join(component_type, "main.nf")] = os.path.join(software_dir, "main.nf") + file_paths[os.path.join(component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") + file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") + file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") + file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") return file_paths diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 083975494e..ef075a20b1 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -1,4 +1,6 @@ import logging +import os +import shutil from pathlib import Path from nf_core.components.components_command import ComponentCommand diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index cfe0b3c557..1284c6d59c 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -59,4 +59,4 @@ def remove(self, module): modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) # Remove the module - return self.clear_component_dir(module_name=module, module_dir=module_dir) + return self.clear_component_dir(module, module_dir) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index bc5125f209..5674ca3106 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -16,6 +16,8 @@ import nf_core.utils from nf_core.modules.module_utils import get_repo_type +from .subworkflows_command import SubworkflowCommand + log = logging.getLogger(__name__) From f17e8f9a15881384cf7ae93c6cdf3f8c3a609c84 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 26 Oct 2022 12:29:26 +0200 Subject: [PATCH 359/854] run tests with Python 3.11 --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 795ebcc789..8e578c8bec 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From bbdf1145a171b77d81b885beee7b6511e88f4465 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 26 Oct 2022 12:32:02 +0200 Subject: [PATCH 360/854] run tests with Python 3.11 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8da17542f..37a1bc5c92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Only warn when checking that the pipeline directory contains a `main.nf` and a `nextflow.config` file if the pipeline is not an nf-core pipeline [#1964](https://github.com/nf-core/tools/pull/1964) - Add file `versions.yml` when generating `test.yml` with `nf-core modules create-test-yml` but don't check for md5sum [#1963](https://github.com/nf-core/tools/pull/1963) - Mock biocontainers and anaconda api calls in modules and subworkflows tests [#1967](https://github.com/nf-core/tools/pull/1967) +- Run tests with Python 3.11 ([#1970](https://github.com/nf-core/tools/pull/1970)) ### Modules From 13ac143f8a2bf3c7c5fee3186f3ac6d33cb7d03d Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 26 Oct 2022 13:03:51 +0200 Subject: [PATCH 361/854] promote the use of Python 3.8 instead of 3.7 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d139286ebe..3513b2f9b1 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ conda install nf-core Alternatively, you can create a new environment with both nf-core/tools and nextflow: ```bash -conda create --name nf-core python=3.7 nf-core nextflow +conda create --name nf-core python=3.8 nf-core nextflow conda activate nf-core ``` From 4b141d965e804eef7fb04ec2e6bae16f711a1521 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 26 Oct 2022 13:09:41 +0200 Subject: [PATCH 362/854] promote use of Python 3.8 instead of 3.7 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a1bc5c92..ca89aeea55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Add file `versions.yml` when generating `test.yml` with `nf-core modules create-test-yml` but don't check for md5sum [#1963](https://github.com/nf-core/tools/pull/1963) - Mock biocontainers and anaconda api calls in modules and subworkflows tests [#1967](https://github.com/nf-core/tools/pull/1967) - Run tests with Python 3.11 ([#1970](https://github.com/nf-core/tools/pull/1970)) +- Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971)) ### Modules From 131e6dd47393687040974ce22727c52c98296c4a Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 27 Oct 2022 00:33:43 +0200 Subject: [PATCH 363/854] refactor prettier invocation --- nf_core/create.py | 38 ++++---------------------------------- nf_core/lint_utils.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index bd00dd1517..5f8383e334 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -10,7 +10,6 @@ import random import re import shutil -import subprocess import sys import time @@ -23,6 +22,7 @@ import nf_core import nf_core.schema import nf_core.utils +from nf_core.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -354,15 +354,7 @@ def update_nextflow_schema(self): schema.get_wf_params() schema.remove_schema_notfound_configs() schema.save_schema(suppress_logging=True) - - # The schema is not guaranteed to follow Prettier standards - # so we run prettier on the schema file - try: - subprocess.run( - ["prettier", "--write", schema_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False - ) - except FileNotFoundError: - log.warning("Prettier not found. Please install it and run it on the pipeline to fix linting issues.") + run_prettier_on_file(schema_path) def remove_nf_core_in_bug_report_template(self): """ @@ -380,17 +372,7 @@ def remove_nf_core_in_bug_report_template(self): with open(bug_report_path, "w") as fh: yaml.dump(contents, fh, default_flow_style=False, sort_keys=False) - # The dumped yaml file will not follow prettier formatting rules - # so we run prettier on the file - try: - subprocess.run( - ["prettier", "--write", bug_report_path], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - except FileNotFoundError: - log.warning("Prettier not found. Please install it and run it on the pipeline to fix linting issues.") + run_prettier_on_file(bug_report_path) def fix_linting(self): """ @@ -458,19 +440,7 @@ def fix_linting(self): with open(os.path.join(self.outdir, ".nf-core.yml"), "w") as fh: yaml.dump(nf_core_yml, fh, default_flow_style=False, sort_keys=False) - # The dumped yaml file will not follow prettier formatting rules - # so we run prettier on the file - try: - subprocess.run( - ["prettier", "--write", os.path.join(self.outdir, ".nf-core.yml")], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - except FileNotFoundError: - log.warning( - "Prettier is not installed. Please install it and run it on the pipeline to fix linting issues." - ) + run_prettier_on_file(os.path.join(self.outdir, ".nf-core.yml")) def make_pipeline_logo(self): """Fetch a logo for the new pipeline from the nf-core website""" diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index ffb3bdf7b3..19d253c53c 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,4 +1,5 @@ import logging +import subprocess import rich from rich.console import Console @@ -47,3 +48,24 @@ def print_fixes(lint_obj, module_lint_obj): console.print( "Automatic fixes applied. Please check with 'git diff' and revert any changes you do not want with 'git checkout '." ) + + +def run_prettier_on_file(file): + """Runs Prettier on a file if Prettier is installed. + + Args: + file (Path | str): A file identifier as a string or pathlib.Path. + + Warns: + If Prettier is not installed, a warning is logged. + """ + + try: + subprocess.run( + ["prettier", "--write", file], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + except FileNotFoundError: + log.warning("Prettier is not installed. Please install it and run it on the pipeline to fix linting issues.") From a03f27b63345fea17db183fa236ed5508d29b356 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 27 Oct 2022 00:40:47 +0200 Subject: [PATCH 364/854] use pathlib Path --- nf_core/create.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 5f8383e334..1fb07ec711 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -6,12 +6,12 @@ import imghdr import logging import os -import pathlib import random import re import shutil import sys import time +from pathlib import Path import git import jinja2 @@ -89,7 +89,7 @@ def __init__( self.force = force if outdir is None: outdir = os.path.join(os.getcwd(), self.template_params["name_noslash"]) - self.outdir = outdir + self.outdir = Path(outdir) def create_param_dict(self, name, description, author, version, template_yaml_path, plain): """Creates a dictionary of parameters for the new pipeline. @@ -246,7 +246,7 @@ def render_template(self): log.info(f"Creating new nf-core pipeline: '{self.name}'") # Check if the output directory exists - if os.path.exists(self.outdir): + if self.outdir.exists(): if self.force: log.warning(f"Output directory '{self.outdir}' exists - continuing as --force specified") else: @@ -265,8 +265,8 @@ def render_template(self): object_attrs["nf_core_version"] = nf_core.__version__ # Can't use glob.glob() as need recursive hidden dotfiles - https://stackoverflow.com/a/58126417/713980 - template_files = list(pathlib.Path(template_dir).glob("**/*")) - template_files += list(pathlib.Path(template_dir).glob("*")) + template_files = list(Path(template_dir).glob("**/*")) + template_files += list(Path(template_dir).glob("*")) ignore_strs = [".pyc", "__pycache__", ".pyo", ".pyd", ".DS_Store", ".egg"] rename_files = { "workflows/pipeline.nf": f"workflows/{self.template_params['short_name']}.nf", @@ -291,9 +291,9 @@ def render_template(self): # Set up vars and directories template_fn = os.path.relpath(template_fn_path, template_dir) - output_path = os.path.join(self.outdir, template_fn) + output_path = self.outdir / template_fn if template_fn in rename_files: - output_path = os.path.join(self.outdir, rename_files[template_fn]) + output_path = self.outdir / rename_files[template_fn] os.makedirs(os.path.dirname(output_path), exist_ok=True) try: @@ -345,7 +345,7 @@ def update_nextflow_schema(self): """ Removes unused parameters from the nextflow schema. """ - schema_path = os.path.join(self.outdir, "nextflow_schema.json") + schema_path = self.outdir / "nextflow_schema.json" schema = nf_core.schema.PipelineSchema() schema.schema_filename = schema_path @@ -361,7 +361,7 @@ def remove_nf_core_in_bug_report_template(self): Remove the field mentioning nf-core documentation in the github bug report template """ - bug_report_path = os.path.join(self.outdir, ".github", "ISSUE_TEMPLATE", "bug_report.yml") + bug_report_path = self.outdir / ".github" / "ISSUE_TEMPLATE" / "bug_report.yml" with open(bug_report_path, "r") as fh: contents = yaml.load(fh, Loader=yaml.FullLoader) @@ -437,7 +437,7 @@ def fix_linting(self): # Add the lint content to the preexisting nf-core config nf_core_yml = nf_core.utils.load_tools_config(self.outdir) nf_core_yml["lint"] = lint_config - with open(os.path.join(self.outdir, ".nf-core.yml"), "w") as fh: + with open(self.outdir / ".nf-core.yml", "w") as fh: yaml.dump(nf_core_yml, fh, default_flow_style=False, sort_keys=False) run_prettier_on_file(os.path.join(self.outdir, ".nf-core.yml")) @@ -448,11 +448,13 @@ def make_pipeline_logo(self): logo_url = f"https://nf-co.re/logo/{self.template_params['short_name']}?theme=light" log.debug(f"Fetching logo from {logo_url}") - email_logo_path = f"{self.outdir}/assets/{self.template_params['name_noslash']}_logo_light.png" + email_logo_path = self.outdir / "assets" / f"{self.template_params['name_noslash']}_logo_light.png" self.download_pipeline_logo(f"{logo_url}&w=400", email_logo_path) for theme in ["dark", "light"]: readme_logo_url = f"{logo_url}?w=600&theme={theme}" - readme_logo_path = f"{self.outdir}/docs/images/{self.template_params['name_noslash']}_logo_{theme}.png" + readme_logo_path = ( + self.outdir / "docs" / "images" / f"{self.template_params['name_noslash']}_logo_{theme}.png" + ) self.download_pipeline_logo(readme_logo_url, readme_logo_path) def download_pipeline_logo(self, url, img_fn): From e0f3b27b97bb80ca3f5b2fba5bde5f943c87148e Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 27 Oct 2022 00:43:51 +0200 Subject: [PATCH 365/854] use implicit sting concatenation and adhere to line length limit --- nf_core/create.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 1fb07ec711..0d26bc5b36 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -236,9 +236,10 @@ def init_pipeline(self): if self.template_params["branded"]: log.info( "[green bold]!!!!!! IMPORTANT !!!!!!\n\n" - + "[green not bold]If you are interested in adding your pipeline to the nf-core community,\n" - + "PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE!\n\n" - + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]https://nf-co.re/developers/adding_pipelines#join-the-community[/link]" + "[green not bold]If you are interested in adding your pipeline to the nf-core community,\n" + "PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE!\n\n" + "[default]Please read: [link=https://nf-co.re/developers/adding_pipelines#join-the-community]" + "https://nf-co.re/developers/adding_pipelines#join-the-community[/link]" ) def render_template(self): From 4031847d598bd50da89e32c8ec5aebac0ab90789 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 27 Oct 2022 00:45:13 +0200 Subject: [PATCH 366/854] use explanatory variable --- nf_core/create.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 0d26bc5b36..feb0e25cbb 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -269,9 +269,10 @@ def render_template(self): template_files = list(Path(template_dir).glob("**/*")) template_files += list(Path(template_dir).glob("*")) ignore_strs = [".pyc", "__pycache__", ".pyo", ".pyd", ".DS_Store", ".egg"] + short_name = self.template_params["short_name"] rename_files = { - "workflows/pipeline.nf": f"workflows/{self.template_params['short_name']}.nf", - "lib/WorkflowPipeline.groovy": f"lib/Workflow{self.template_params['short_name'][0].upper()}{self.template_params['short_name'][1:]}.groovy", + "workflows/pipeline.nf": f"workflows/{short_name}.nf", + "lib/WorkflowPipeline.groovy": f"lib/Workflow{short_name[0].upper()}{short_name[1:]}.groovy", } # Set the paths to skip according to customization From 215f96e78992ecb4dd4e8d4b80ea5671aa925703 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 27 Oct 2022 00:51:21 +0200 Subject: [PATCH 367/854] sort requirements --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8fc8d1c7de..6d44ea1bc9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,6 +4,6 @@ myst_parser pre-commit pytest-cov pytest-datafiles +requests-mock Sphinx sphinx_rtd_theme -requests_mock From ecd9d1f23510ad910bddc939ad54175144bc9e67 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 27 Oct 2022 00:51:59 +0200 Subject: [PATCH 368/854] standardize requirements format --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6d44ea1bc9..4989f23744 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,4 @@ pytest-cov pytest-datafiles requests-mock Sphinx -sphinx_rtd_theme +sphinx-rtd-theme From a0becb70d71f485ccaa92588a9ab276625ec4bb5 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 12 Oct 2022 09:08:14 +0200 Subject: [PATCH 369/854] Check outdated version is notified to the user --- tests/test_cli.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index a98fe8a407..27b39127bd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,6 +15,15 @@ def test_header(mock_cli): nf_core.__main__.run_nf_core() +@mock.patch("nf_core.__main__.nf_core_cli") +@mock.patch("nf_core.utils.check_if_outdated", return_value=(True, None, "dummy_version")) +def test_header_outdated(mock_check_outdated, mock_nf_core_cli, capsys): + """Check cli notifies the user when nf_core is outdated""" + nf_core.__main__.run_nf_core() + captured = capsys.readouterr() + assert "There is a new version of nf-core/tools available! (dummy_version)" in captured.err + + def test_cli_help(): """Test the main launch function with --help""" runner = CliRunner() From ea1ddd55b99624148d9de165e0871290d7490d6a Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 12 Oct 2022 09:29:06 +0200 Subject: [PATCH 370/854] Test nf-core list --- tests/test_cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 27b39127bd..f4403cb97d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -39,3 +39,14 @@ def test_cli_bad_subcommand(): assert result.exit_code == 2 # Checks that -v was considered valid assert "No such command" in result.output + + +@mock.patch("nf_core.list.list_workflows", return_value="pipeline test list") +def test_cli_list(mock_list_workflows): + """Test nf-core pipelines are listed and cli parameters are passed on.""" + runner = CliRunner() + result = runner.invoke( + nf_core.__main__.nf_core_cli, ["list", "--sort", "name", "--json", "--show-archived", "kw1", "kw2"] + ) + mock_list_workflows.assert_called_once_with(("kw1", "kw2"), "name", True, True) + assert "pipeline test list" in result.output From d7740d86072f1298064aa3600dfaffbdc2770f33 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 12 Oct 2022 11:20:02 +0200 Subject: [PATCH 371/854] Group cli tests into a class --- tests/test_cli.py | 81 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index f4403cb97d..dab4c13478 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,8 @@ """ Tests covering the command-line code. """ +import tempfile +import unittest from unittest import mock from click.testing import CliRunner @@ -24,29 +26,56 @@ def test_header_outdated(mock_check_outdated, mock_nf_core_cli, capsys): assert "There is a new version of nf-core/tools available! (dummy_version)" in captured.err -def test_cli_help(): - """Test the main launch function with --help""" - runner = CliRunner() - result = runner.invoke(nf_core.__main__.nf_core_cli, ["--help"]) - assert result.exit_code == 0 - assert "Show the version and exit." in result.output - - -def test_cli_bad_subcommand(): - """Test the main launch function with verbose flag and an unrecognised argument""" - runner = CliRunner() - result = runner.invoke(nf_core.__main__.nf_core_cli, ["-v", "foo"]) - assert result.exit_code == 2 - # Checks that -v was considered valid - assert "No such command" in result.output - - -@mock.patch("nf_core.list.list_workflows", return_value="pipeline test list") -def test_cli_list(mock_list_workflows): - """Test nf-core pipelines are listed and cli parameters are passed on.""" - runner = CliRunner() - result = runner.invoke( - nf_core.__main__.nf_core_cli, ["list", "--sort", "name", "--json", "--show-archived", "kw1", "kw2"] - ) - mock_list_workflows.assert_called_once_with(("kw1", "kw2"), "name", True, True) - assert "pipeline test list" in result.output +class TestCli(unittest.TestCase): + """Class for testing the commandline interface""" + + def setUp(self): + self.runner = CliRunner() + + def assemble_params(self, params): + """Assemble a dictionnary of parameters into a list of arguments for the cli + + Note: + if the value of a parameter is None, it will be considered a flag + + Args: + params (dict): dict of parameters to assemble""" + arg_list = [[f"--{key}"] + ([value] if value is not None else []) for key, value in params.items()] + return [item for arg in arg_list for item in arg] + + def invoke_cli(self, cmd): + """Invoke the commandline interface using a list of parameters + + Args: + cmd (list): commandline to execute + """ + return self.runner.invoke(nf_core.__main__.nf_core_cli, cmd) + + def test_cli_help(self): + """Test the main launch function with --help""" + result = self.invoke_cli(["--help"]) + assert result.exit_code == 0 + assert "Show the version and exit." in result.output + + def test_cli_bad_subcommand(self): + """Test the main launch function with verbose flag and an unrecognised argument""" + result = self.invoke_cli(["-v", "foo"]) + assert result.exit_code == 2 + # Checks that -v was considered valid + assert "No such command" in result.output + + @mock.patch("nf_core.list.list_workflows", return_value="pipeline test list") + def test_cli_list(self, mock_list_workflows): + """Test nf-core pipelines are listed and cli parameters are passed on.""" + params = { + "sort": "name", + "json": None, + "show-archived": None, + } + cmd = ["list"] + self.assemble_params(params) + ["kw1", "kw2"] + result = self.invoke_cli(cmd) + + mock_list_workflows.assert_called_once_with( + tuple(cmd[-2:]), params["sort"], "json" in params, "show-archived" in params + ) + assert "pipeline test list" in result.output From 68e50631e2410fc9cf2f00dc47bdae635d8ed8f4 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 12 Oct 2022 17:06:11 +0200 Subject: [PATCH 372/854] Test nf-core launch --- tests/test_cli.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index dab4c13478..debdaefcba 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -79,3 +79,62 @@ def test_cli_list(self, mock_list_workflows): tuple(cmd[-2:]), params["sort"], "json" in params, "show-archived" in params ) assert "pipeline test list" in result.output + + @mock.patch("nf_core.launch.Launch") + def test_cli_launch(self, mock_launcher): + """Test nf-core pipeline is launched and cli parameters are passed on.""" + mock_launcher.return_value.launch_pipeline.return_value = True + + temp_params_in = tempfile.NamedTemporaryFile() + params = { + "revision": "abcdef", + "id": "idgui", + "command-only": None, + "params-out": "/path/params/out", + "params-in": temp_params_in.name, + "save-all": None, + "show-hidden": None, + "url": "builder_url", + } + cmd = ["launch"] + self.assemble_params(params) + ["pipeline_name"] + result = self.invoke_cli(cmd) + + assert result.exit_code == 0 + + mock_launcher.assert_called_once_with( + cmd[-1], + params["revision"], + "command-only" in params, + params["params-in"], + params["params-out"], + "save-all" in params, + "show-hidden" in params, + params["url"], + params["id"], + ) + + mock_launcher.return_value.launch_pipeline.assert_called_once() + + @mock.patch("nf_core.launch.Launch") + def test_cli_launch_no_params_in(self, mock_launcher): + """Test nf-core pipeline fails when params-in does not exist""" + mock_launcher.return_value.launch_pipeline.return_value = True + + params = { + "params-in": "/fake/path", + } + cmd = ["launch"] + self.assemble_params(params) + ["pipeline_name"] + result = self.invoke_cli(cmd) + + assert result.exit_code == 2 + assert f"Invalid value for '-p' / '--params-in': Path '{params['params-in']}' does not exist." in result.output + + mock_launcher.assert_not_called() + + @mock.patch("nf_core.launch.Launch") + def test_cli_launch_fail(self, mock_launcher): + """Test nf-core pipeline fails with exit code 1 when pipeline fails.""" + mock_launcher.return_value.launch_pipeline.return_value = False + cmd = ["launch", "pipeline_name"] + result = self.invoke_cli(cmd) + assert result.exit_code == 1 From d3a501925cad6be8063148763c1237e8f981c3dc Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 12 Oct 2022 17:08:13 +0200 Subject: [PATCH 373/854] Check nf-core list returns 0 code --- tests/test_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index debdaefcba..240f255898 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -78,6 +78,7 @@ def test_cli_list(self, mock_list_workflows): mock_list_workflows.assert_called_once_with( tuple(cmd[-2:]), params["sort"], "json" in params, "show-archived" in params ) + assert result.exit_code == 0 assert "pipeline test list" in result.output @mock.patch("nf_core.launch.Launch") From 33d3f3f3048b64c6ecd0415312d62238614063b4 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Fri, 21 Oct 2022 13:58:22 +0200 Subject: [PATCH 374/854] Improve test documentation --- tests/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 240f255898..94df9dca49 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,8 @@ #!/usr/bin/env python """ Tests covering the command-line code. + +Most tests check the cli arguments are passed along and that some action is +taken. """ import tempfile From 151c6e8ef21638137fa91568fb8b68b5d084a765 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Fri, 21 Oct 2022 14:00:14 +0200 Subject: [PATCH 375/854] Test nf-core download --- tests/test_cli.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 94df9dca49..1d03f3303a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -142,3 +142,34 @@ def test_cli_launch_fail(self, mock_launcher): cmd = ["launch", "pipeline_name"] result = self.invoke_cli(cmd) assert result.exit_code == 1 + + @mock.patch("nf_core.download.DownloadWorkflow") + def test_cli_download(self, mock_dl): + """Test nf-core pipeline is downloaded and cli parameters are passed on.""" + params = { + "revision": "abcdef", + "outdir": "/path/outdir", + "compress": "tar.gz", + "force": None, + "container": "singularity", + "singularity-cache-only": None, + "parallel-downloads": 2, + } + + cmd = ["download"] + self.assemble_params(params) + ["pipeline_name"] + result = self.invoke_cli(cmd) + + assert result.exit_code == 0 + + mock_dl.assert_called_once_with( + cmd[-1], + params["revision"], + params["outdir"], + params["compress"], + "force" in params, + params["container"], + "singularity-cache-only" in params, + params["parallel-downloads"], + ) + + mock_dl.return_value.download_workflow.assert_called_once() From 226f0b4a9a703b75f4cd41b249cc4e07e08bd427 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Fri, 21 Oct 2022 14:46:34 +0200 Subject: [PATCH 376/854] Test nf-core create --- tests/test_cli.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d03f3303a..bdc4251d5e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -39,7 +39,8 @@ def assemble_params(self, params): """Assemble a dictionnary of parameters into a list of arguments for the cli Note: - if the value of a parameter is None, it will be considered a flag + if the value of a parameter is None, it will be considered a flag. + Booleans were not used to avoid conflicting with the click.BOOL type. Args: params (dict): dict of parameters to assemble""" @@ -173,3 +174,67 @@ def test_cli_download(self, mock_dl): ) mock_dl.return_value.download_workflow.assert_called_once() + + @mock.patch("nf_core.licences.WorkflowLicences") + def test_licences(self, mock_lic): + """Test nf-core pipeline licence is printed out and cli parameters are passed on.""" + licence_text = "dummy licence text" + mock_lic.return_value.run_licences.return_value = licence_text + + params = { + "json": None, + } + + cmd = ["licences"] + self.assemble_params(params) + ["pipeline_name"] + result = self.invoke_cli(cmd) + + assert result.exit_code == 0 + assert licence_text in result.output + + mock_lic.assert_called_once_with(cmd[-1]) + + @mock.patch("nf_core.licences.WorkflowLicences") + def test_licences_log_error(self, mock_lic): + """Test LookupError is logged""" + error_txt = "LookupError has been raised" + mock_lic.return_value.run_licences.side_effect = LookupError(error_txt) + + cmd = ["licences", "pipeline_name"] + with self.assertLogs() as captured_logs: + result = self.invoke_cli(cmd) + + assert result.exit_code == 1 + assert error_txt in captured_logs.output[-1] + assert captured_logs.records[-1].levelname == "ERROR" + + @mock.patch("nf_core.create.PipelineCreate") + def test_create(self, mock_create): + """Test nf-core pipeline is created and cli parameters are passed on.""" + params = { + "name": "pipeline name", + "description": "pipeline description", + "author": "Kalle Anka", + "version": "1.2.3", + "no-git": None, + "force": None, + "outdir": "/path/outdir", + "template-yaml": "file.yaml", + "plain": None, + } + + cmd = ["create"] + self.assemble_params(params) + result = self.invoke_cli(cmd) + + assert result.exit_code == 0 + mock_create.assert_called_once_with( + params["name"], + params["description"], + params["author"], + params["version"], + "no-git" in params, + "force" in params, + params["outdir"], + params["template-yaml"], + "plain" in params, + ) + mock_create.return_value.init_pipeline.assert_called_once() From ee58097532cdab0ad78ed7d87743171cf002bf0f Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Fri, 21 Oct 2022 16:12:49 +0200 Subject: [PATCH 377/854] Test nf-core lint --- tests/test_cli.py | 96 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index bdc4251d5e..7c3048dfed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -238,3 +238,99 @@ def test_create(self, mock_create): "plain" in params, ) mock_create.return_value.init_pipeline.assert_called_once() + + @mock.patch("nf_core.utils.is_pipeline_directory") + @mock.patch("nf_core.lint.run_linting") + def test_lint(self, mock_lint, mock_is_pipeline): + """Test nf-core lint""" + mock_lint_results = (mock.MagicMock, mock.MagicMock) + mock_lint_results[0].failed = [] + mock_lint_results[1].failed = [] + mock_lint.return_value = mock_lint_results + + temp_pipeline_dir = tempfile.NamedTemporaryFile() + params = { + "dir": temp_pipeline_dir.name, + "release": None, + "fix": "fix test", + "key": "key test", + "show-passed": None, + "fail-ignored": None, + "fail-warned": None, + "markdown": "output_file.md", + "json": "output_file.json", + "hide-progress": None, + } + + cmd = ["lint"] + self.assemble_params(params) + result = self.invoke_cli(cmd) + + assert result.exit_code == 0 + mock_lint.assert_called_once_with( + params["dir"], + "release" in params, + (params["fix"],), + (params["key"],), + "show-passed" in params, + "fail-ignored" in params, + "fail-warned" in params, + params["markdown"], + params["json"], + "hide-progress" in params, + ) + + def test_lint_no_dir(self): + """Test nf-core lint fails if --dir does not exist""" + params = { + "dir": "/bad/path", + } + + cmd = ["lint"] + self.assemble_params(params) + result = self.invoke_cli(cmd) + + assert result.exit_code == 2 + assert f"Invalid value for '-d' / '--dir': Path '{params['dir']}' does not exist." in result.output + + @mock.patch("nf_core.utils.is_pipeline_directory") + def test_lint_dir_is_not_pipeline(self, mock_is_pipeline): + """Test nf-core lint logs an error if not called from a pipeline directory.""" + error_txt = "UserWarning has been raised" + mock_is_pipeline.side_effect = UserWarning(error_txt) + + cmd = ["lint"] + with self.assertLogs() as captured_logs: + result = self.invoke_cli(cmd) + + assert result.exit_code == 1 + assert error_txt in captured_logs.output[-1] + assert captured_logs.records[-1].levelname == "ERROR" + + @mock.patch("nf_core.utils.is_pipeline_directory") + @mock.patch("nf_core.lint.run_linting") + def test_lint_log_assert_error(self, mock_lint, mock_is_pipeline): + """Test nf-core lint logs assertion errors""" + error_txt = "AssertionError has been raised" + mock_lint.side_effect = AssertionError(error_txt) + + cmd = ["lint"] + with self.assertLogs() as captured_logs: + result = self.invoke_cli(cmd) + + assert result.exit_code == 1 + assert error_txt in captured_logs.output[-1] + assert captured_logs.records[-1].levelname == "CRITICAL" + + @mock.patch("nf_core.utils.is_pipeline_directory") + @mock.patch("nf_core.lint.run_linting") + def test_lint_log_assert_error(self, mock_lint, mock_is_pipeline): + """Test nf-core lint logs assertion errors""" + error_txt = "AssertionError has been raised" + mock_lint.side_effect = UserWarning(error_txt) + + cmd = ["lint"] + with self.assertLogs() as captured_logs: + result = self.invoke_cli(cmd) + + assert result.exit_code == 1 + assert error_txt in captured_logs.output[-1] + assert captured_logs.records[-1].levelname == "ERROR" From 4fbb15a3c960a9030dd7613137980c77bdb95b60 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Mon, 24 Oct 2022 16:18:48 +0200 Subject: [PATCH 378/854] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca89aeea55..521030cbf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ - Don't remove local copy of modules repo, only update it with fetch ([#1881](https://github.com/nf-core/tools/pull/1881)) - Add subworkflow commands create-test-yml, create and install ([#1897](https://github.com/nf-core/tools/pull/1897)) - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) -- Improve test coverage of sync.py +- Improve test coverage of `sync.py` and `__main__.py` ([#1936](https://github.com/nf-core/tools/pull/1936), [#1965](https://github.com/nf-core/tools/pull/1965)) - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). - Only warn when checking that the pipeline directory contains a `main.nf` and a `nextflow.config` file if the pipeline is not an nf-core pipeline [#1964](https://github.com/nf-core/tools/pull/1964) From a2a9b396f6da244e765cf8d5fe44e7d5e077b8c0 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 25 Oct 2022 12:58:58 +0200 Subject: [PATCH 379/854] Strip ansi codes fron cli output --- tests/test_cli.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7c3048dfed..c7dfba5928 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,6 +12,7 @@ from click.testing import CliRunner import nf_core.__main__ +import nf_core.utils @mock.patch("nf_core.__main__.nf_core_cli") @@ -132,7 +133,10 @@ def test_cli_launch_no_params_in(self, mock_launcher): result = self.invoke_cli(cmd) assert result.exit_code == 2 - assert f"Invalid value for '-p' / '--params-in': Path '{params['params-in']}' does not exist." in result.output + assert ( + f"Invalid value for '-p' / '--params-in': Path '{params['params-in']}' does not exist." + in nf_core.utils.strip_ansi_codes(result.output) + ) mock_launcher.assert_not_called() @@ -289,7 +293,10 @@ def test_lint_no_dir(self): result = self.invoke_cli(cmd) assert result.exit_code == 2 - assert f"Invalid value for '-d' / '--dir': Path '{params['dir']}' does not exist." in result.output + assert ( + f"Invalid value for '-d' / '--dir': Path '{params['dir']}' does not exist." + in nf_core.utils.strip_ansi_codes(result.output) + ) @mock.patch("nf_core.utils.is_pipeline_directory") def test_lint_dir_is_not_pipeline(self, mock_is_pipeline): From c86091aec368805117742877b52763f37325da61 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 25 Oct 2022 13:55:27 +0200 Subject: [PATCH 380/854] Add deprecation warning to nf-core licences --- nf_core/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b49693a5c4..479ce85741 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -253,12 +253,14 @@ def download(pipeline, revision, outdir, compress, force, container, singularity @click.option("--json", is_flag=True, default=False, help="Print output in JSON") def licences(pipeline, json): """ - List software licences for a given workflow. + List software licences for a given workflow (deprecated). Checks the pipeline environment.yml file which lists all conda software packages. Each of these is queried against the anaconda.org API to find the licence. Package name, version and licence is printed to the command line. """ + log.warning("`nf-core licences` does not support DSL2 pipelines and is being deprecated.") + lic = nf_core.licences.WorkflowLicences(pipeline) lic.as_json = json try: From cb4cc9c1dd468da692d710bcd4d29dcc3e3a8572 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 25 Oct 2022 14:14:08 +0200 Subject: [PATCH 381/854] Fix duplicate test name --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index c7dfba5928..090661c83e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -329,7 +329,7 @@ def test_lint_log_assert_error(self, mock_lint, mock_is_pipeline): @mock.patch("nf_core.utils.is_pipeline_directory") @mock.patch("nf_core.lint.run_linting") - def test_lint_log_assert_error(self, mock_lint, mock_is_pipeline): + def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): """Test nf-core lint logs assertion errors""" error_txt = "AssertionError has been raised" mock_lint.side_effect = UserWarning(error_txt) From 2b5440b98987297159c6ffeca30a3c5729e38fc8 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Thu, 27 Oct 2022 11:28:54 +0200 Subject: [PATCH 382/854] Split test cli bad command into two tests --- tests/test_cli.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 090661c83e..3fba1ff595 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -63,11 +63,15 @@ def test_cli_help(self): assert "Show the version and exit." in result.output def test_cli_bad_subcommand(self): - """Test the main launch function with verbose flag and an unrecognised argument""" - result = self.invoke_cli(["-v", "foo"]) + """Test the main launch function with an unrecognised argument""" + result = self.invoke_cli(["foo"]) assert result.exit_code == 2 + + def test_cli_verbose(self): + """Test the main launch function with verbose flag""" + result = self.invoke_cli(["-v"]) # Checks that -v was considered valid - assert "No such command" in result.output + assert "No such option: -v" not in nf_core.utils.strip_ansi_codes(result.output) @mock.patch("nf_core.list.list_workflows", return_value="pipeline test list") def test_cli_list(self, mock_list_workflows): From 91efa5e052c7c7943ec89b34a30c3b7e13bfb028 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Thu, 27 Oct 2022 11:34:56 +0200 Subject: [PATCH 383/854] Use for-loop instead of list-comprehension for more clarity --- tests/test_cli.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3fba1ff595..d8d9eb29c7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -45,8 +45,14 @@ def assemble_params(self, params): Args: params (dict): dict of parameters to assemble""" - arg_list = [[f"--{key}"] + ([value] if value is not None else []) for key, value in params.items()] - return [item for arg in arg_list for item in arg] + arg_list = [] + for key, value in params.items(): + if value is not None: + arg_list += [f"--{key}", value] + else: + arg_list += [f"--{key}"] + + return arg_list def invoke_cli(self, cmd): """Invoke the commandline interface using a list of parameters From b76d3cdf9b7839f2412de90ff9ef7b70473e2a16 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Thu, 27 Oct 2022 14:07:57 +0200 Subject: [PATCH 384/854] Highlight `licences` only supports DSL1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/__main__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 479ce85741..3e9fd73e95 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -253,13 +253,12 @@ def download(pipeline, revision, outdir, compress, force, container, singularity @click.option("--json", is_flag=True, default=False, help="Print output in JSON") def licences(pipeline, json): """ - List software licences for a given workflow (deprecated). + List software licences for a given workflow (DSL1 only). - Checks the pipeline environment.yml file which lists all conda software packages. + Checks the pipeline environment.yml file which lists all conda software packages, which is not available for DSL2 workflows. Therefore, this command only supports DSL1 workflows (for now). Each of these is queried against the anaconda.org API to find the licence. Package name, version and licence is printed to the command line. """ - log.warning("`nf-core licences` does not support DSL2 pipelines and is being deprecated.") lic = nf_core.licences.WorkflowLicences(pipeline) lic.as_json = json From f1b42ab77ec9f1a21fee45369abf642e4678a6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 27 Oct 2022 15:12:31 +0200 Subject: [PATCH 385/854] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 231e69261a..6eabb960f3 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -66,7 +66,7 @@ def has_valid_directory(self): if self.repo_type == "modules": return True if self.dir is None or not os.path.exists(self.dir): - log.error(f"Could not find pipeline: {self.dir}") + log.error(f"Could not find directory: {self.dir}") return False main_nf = os.path.join(self.dir, "main.nf") nf_config = os.path.join(self.dir, "nextflow.config") From 4d96ee5a177055aae85922fa975a037470581c0c Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Thu, 27 Oct 2022 14:14:50 +0100 Subject: [PATCH 386/854] Missing subprocess import --- nf_core/components/components_create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py index 6b2962498c..1206c7c415 100644 --- a/nf_core/components/components_create.py +++ b/nf_core/components/components_create.py @@ -3,6 +3,7 @@ import logging import os import re +import subprocess import jinja2 import rich From 2bac8fadff54244fae828d411fbcc66c135a0355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 27 Oct 2022 14:24:19 +0000 Subject: [PATCH 387/854] change module_utils.py to modules_utils.py --- CHANGELOG.md | 2 +- nf_core/__main__.py | 2 +- nf_core/modules/__init__.py | 2 +- nf_core/modules/bump_versions.py | 12 ++++++------ nf_core/modules/create.py | 4 ++-- nf_core/modules/info.py | 2 +- nf_core/modules/install.py | 4 ++-- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/lint/main_nf.py | 2 +- nf_core/modules/lint/module_version.py | 2 +- nf_core/modules/list.py | 4 ++-- nf_core/modules/module_test.py | 2 +- nf_core/modules/modules_json.py | 18 +++++++++--------- nf_core/modules/modules_repo.py | 6 +++--- .../{module_utils.py => modules_utils.py} | 0 nf_core/modules/update.py | 4 ++-- nf_core/subworkflows/create.py | 2 +- nf_core/subworkflows/install.py | 4 ++-- nf_core/subworkflows/subworkflows_test.py | 2 +- tests/modules/bump_versions.py | 2 +- 20 files changed, 39 insertions(+), 39 deletions(-) rename nf_core/modules/{module_utils.py => modules_utils.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8da17542f..927de9bdcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -363,7 +363,7 @@ Please note that there are many excellent integrations for Prettier available, f ### Modules -- Fixed typo in `module_utils.py`. +- Fixed typo in `modules_utils.py`. - Fixed failing lint test when process section was missing from module. Also added the local failing tests to the warned section of the output table. ([#1235](https://github.com/nf-core/tools/issues/1235)) - Added `--diff` flag to `nf-core modules update` which shows the diff between the installed files and the versions - Update `nf-core modules create` help texts which were not changed with the introduction of the `--dir` flag diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b49693a5c4..b8ae011253 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -827,7 +827,7 @@ def bump_versions(ctx, tool, dir, all, show_all): ctx.obj["modules_repo_no_pull"], ) version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all) - except nf_core.modules.module_utils.ModuleException as e: + except nf_core.modules.modules_utils.ModuleException as e: log.error(e) sys.exit(1) except (UserWarning, LookupError) as e: diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index ad3306ceef..6069e6b2d4 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -5,7 +5,7 @@ from .lint import ModuleLint from .list import ModuleList from .module_test import ModulesTest -from .module_utils import ModuleException +from .modules_utils import ModuleException from .modules_json import ModulesJson from .modules_repo import ModulesRepo from .mulled import MulledImageNameGenerator diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index fc537c3713..5d064cf2ef 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -16,7 +16,7 @@ from rich.markdown import Markdown from rich.table import Table -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.utils import plural_s as _s from nf_core.utils import rich_force_colors @@ -59,14 +59,14 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self.check_modules_structure() # Verify that this is not a pipeline - self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir) if not repo_type == "modules": - raise nf_core.modules.module_utils.ModuleException( + raise nf_core.modules.modules_utils.ModuleException( "This command only works on the nf-core/modules repository, not on pipelines!" ) # Get list of all modules - _, nfcore_modules = nf_core.modules.module_utils.get_installed_modules(self.dir) + _, nfcore_modules = nf_core.modules.modules_utils.get_installed_modules(self.dir) # Load the .nf-core.yml config self.tools_config = nf_core.utils.load_tools_config(self.dir) @@ -92,12 +92,12 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): if module: self.show_up_to_date = True if all_modules: - raise nf_core.modules.module_utils.ModuleException( + raise nf_core.modules.modules_utils.ModuleException( "You cannot specify a tool and request all tools to be bumped." ) nfcore_modules = [m for m in nfcore_modules if m.module_name == module] if len(nfcore_modules) == 0: - raise nf_core.modules.module_utils.ModuleException(f"Could not find the specified module: '{module}'") + raise nf_core.modules.modules_utils.ModuleException(f"Could not find the specified module: '{module}'") progress_bar = rich.progress.Progress( "[bold blue]{task.description}", diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 2828fd6aea..1b1eeba5ea 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -15,7 +15,7 @@ import nf_core import nf_core.components.components_create -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from .modules_command import ModuleCommand @@ -89,7 +89,7 @@ def create(self): # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: - self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type) + self.directory, self.repo_type = nf_core.modules.modules_utils.get_repo_type(self.directory, self.repo_type) except LookupError as e: raise UserWarning(e) log.info(f"Repository type: [blue]{self.repo_type}") diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index a900d4b501..635b8eba41 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -13,7 +13,7 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson -from .module_utils import get_repo_type +from .modules_utils import get_repo_type from .modules_command import ModuleCommand from .modules_repo import NF_CORE_MODULES_REMOTE diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index a787ac8077..70423a4771 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -3,7 +3,7 @@ import questionary -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.modules_json import ModulesJson @@ -109,7 +109,7 @@ def install(self, module, silent=False): version = self.sha elif self.prompt: try: - version = nf_core.modules.module_utils.prompt_module_version_sha( + version = nf_core.modules.modules_utils.prompt_module_version_sha( module, installed_sha=current_version, modules_repo=self.modules_repo, diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 9cd000acdd..f4ab5583ca 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -19,7 +19,7 @@ from rich.markdown import Markdown from rich.table import Table -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.lint_utils import console from nf_core.modules.modules_command import ModuleCommand diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 46dae14d04..e396e7e813 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -11,7 +11,7 @@ import requests import nf_core -import nf_core.modules.module_utils +import nf_core.modules.modules_utils from nf_core.modules.modules_differ import ModulesDiffer log = logging.getLogger(__name__) diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index 9077cc5371..52df63f06b 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -7,7 +7,7 @@ from pathlib import Path import nf_core -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.modules.modules_repo log = logging.getLogger(__name__) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index a7725a70cc..ea9c69c4a6 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -3,7 +3,7 @@ import rich -import nf_core.modules.module_utils +import nf_core.modules.modules_utils from .modules_command import ModuleCommand from .modules_json import ModulesJson @@ -67,7 +67,7 @@ def pattern_msg(keywords): else: # Check that we are in a pipeline directory try: - _, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + _, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir) if repo_type != "pipeline": raise UserWarning( "The command 'nf-core modules list local' must be run from a pipeline directory.", diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 10b83ad853..92227b992b 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -13,7 +13,7 @@ import questionary import rich -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.modules_command import ModuleCommand from nf_core.modules.modules_json import ModulesJson diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index a095049e65..615b35d98c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -11,7 +11,7 @@ import questionary from git.exc import GitCommandError -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.modules.modules_repo import nf_core.utils @@ -64,15 +64,15 @@ def create(self): [ str( Path(module_name).relative_to( - modules_dir / nf_core.modules.module_utils.path_from_remote(repo_url) + modules_dir / nf_core.modules.modules_utils.path_from_remote(repo_url) ) ) for module_name, _, file_names in os.walk( - modules_dir / nf_core.modules.module_utils.path_from_remote(repo_url) + modules_dir / nf_core.modules.modules_utils.path_from_remote(repo_url) ) if "main.nf" in file_names ], - nf_core.modules.module_utils.path_from_remote(repo_url), + nf_core.modules.modules_utils.path_from_remote(repo_url), ) for repo_url in repos ] @@ -113,7 +113,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): renamed_dirs = {} # Check if there are any untracked repositories dirs_not_covered = self.dir_tree_uncovered( - modules_dir, [Path(nf_core.modules.module_utils.path_from_remote(url)) for url in repos] + modules_dir, [Path(nf_core.modules.modules_utils.path_from_remote(url)) for url in repos] ) if len(dirs_not_covered) > 0: log.info("Found custom module repositories when creating 'modules.json'") @@ -140,7 +140,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): ).unsafe_ask() # Verify that there is a directory corresponding the remote - nrepo_name = nf_core.modules.module_utils.path_from_remote(nrepo_remote) + nrepo_name = nf_core.modules.modules_utils.path_from_remote(nrepo_remote) if not (modules_dir / nrepo_name).exists(): log.info( "The provided remote does not seem to correspond to a local directory. " @@ -901,12 +901,12 @@ def resolve_missing_from_modules_json(self, missing_from_modules_json, component components_with_repos = ( ( - nf_core.modules.module_utils.path_from_remote(repo_url), - str(dir.relative_to(nf_core.modules.module_utils.path_from_remote(repo_url))), + nf_core.modules.modules_utils.path_from_remote(repo_url), + str(dir.relative_to(nf_core.modules.modules_utils.path_from_remote(repo_url))), ) for dir in missing_from_modules_json for repo_url in repos - if nf_core.utils.is_relative_to(dir, nf_core.modules.module_utils.path_from_remote(repo_url)) + if nf_core.utils.is_relative_to(dir, nf_core.modules.modules_utils.path_from_remote(repo_url)) ) repos_with_components = {} diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 5621a02249..250446d012 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -10,7 +10,7 @@ import rich.progress from git.exc import GitCommandError, InvalidGitRepositoryError -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.modules.modules_json from nf_core.utils import NFCORE_DIR @@ -127,8 +127,8 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url - self.repo_path = nf_core.modules.module_utils.path_from_remote(self.remote_url) - self.fullname = nf_core.modules.module_utils.repo_full_name_from_remote(self.remote_url) + self.repo_path = nf_core.modules.modules_utils.path_from_remote(self.remote_url) + self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/modules_utils.py similarity index 100% rename from nf_core/modules/module_utils.py rename to nf_core/modules/modules_utils.py diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index ce39b4b765..cd39a4565f 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -6,7 +6,7 @@ import questionary -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.utils import plural_es, plural_s, plural_y @@ -151,7 +151,7 @@ def update(self, module=None): if sha is not None: version = sha elif self.prompt: - version = nf_core.modules.module_utils.prompt_module_version_sha( + version = nf_core.modules.modules_utils.prompt_module_version_sha( module, modules_repo=modules_repo, installed_sha=current_version ) else: diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 5674ca3106..7181498ef5 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -14,7 +14,7 @@ import nf_core import nf_core.components.components_create import nf_core.utils -from nf_core.modules.module_utils import get_repo_type +from nf_core.modules.modules_utils import get_repo_type from .subworkflows_command import SubworkflowCommand diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index f0959e92d9..81ba059ae4 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -5,7 +5,7 @@ import questionary -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson @@ -107,7 +107,7 @@ def install(self, subworkflow, silent=False): version = self.sha elif self.prompt: try: - version = nf_core.modules.module_utils.prompt_module_version_sha( + version = nf_core.modules.modules_utils.prompt_module_version_sha( subworkflow, installed_sha=current_version, modules_repo=self.modules_repo, diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 82f2b1441f..06395080dd 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -13,7 +13,7 @@ import questionary import rich -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.modules_json import ModulesJson diff --git a/tests/modules/bump_versions.py b/tests/modules/bump_versions.py index e8b5803f00..65569efd51 100644 --- a/tests/modules/bump_versions.py +++ b/tests/modules/bump_versions.py @@ -4,7 +4,7 @@ import pytest import nf_core.modules -from nf_core.modules.module_utils import ModuleException +from nf_core.modules.modules_utils import ModuleException def test_modules_bump_versions_single_module(self): From 76191419e88045c236a4c95c2b06aa7bbec961b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 27 Oct 2022 14:32:58 +0000 Subject: [PATCH 388/854] change module_test.py to modules_test.py --- nf_core/modules/__init__.py | 2 +- nf_core/modules/{module_test.py => modules_test.py} | 0 tests/modules/{module_test.py => modules_test.py} | 0 tests/test_modules.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename nf_core/modules/{module_test.py => modules_test.py} (100%) rename tests/modules/{module_test.py => modules_test.py} (100%) diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index 6069e6b2d4..c60c630fc0 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -4,7 +4,7 @@ from .install import ModuleInstall from .lint import ModuleLint from .list import ModuleList -from .module_test import ModulesTest +from .modules_test import ModulesTest from .modules_utils import ModuleException from .modules_json import ModulesJson from .modules_repo import ModulesRepo diff --git a/nf_core/modules/module_test.py b/nf_core/modules/modules_test.py similarity index 100% rename from nf_core/modules/module_test.py rename to nf_core/modules/modules_test.py diff --git a/tests/modules/module_test.py b/tests/modules/modules_test.py similarity index 100% rename from tests/modules/module_test.py rename to tests/modules/modules_test.py diff --git a/tests/test_modules.py b/tests/test_modules.py index 157de39979..9cbab99801 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -152,7 +152,7 @@ def test_modulesrepo_class(self): test_modules_list_remote, test_modules_list_remote_gitlab, ) - from .modules.module_test import ( + from .modules.modules_test import ( test_modules_test_check_inputs, test_modules_test_no_installed_modules, test_modules_test_no_name_no_prompts, From 924ef527be4969e62df49d3c667a55dfb686b6fe Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 27 Oct 2022 16:34:44 +0200 Subject: [PATCH 389/854] add tests for `subworkflows test` --- nf_core/components/components_test.py | 158 ++++++++++++++++++++++ nf_core/subworkflows/__init__.py | 1 + nf_core/subworkflows/subworkflows_test.py | 12 +- tests/subworkflows/subworkflows_test.py | 41 ++++++ tests/test_subworkflows.py | 6 + 5 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 nf_core/components/components_test.py create mode 100644 tests/subworkflows/subworkflows_test.py diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py new file mode 100644 index 0000000000..e8a600f785 --- /dev/null +++ b/nf_core/components/components_test.py @@ -0,0 +1,158 @@ +class ComponentsTest(ComponentCommand): + """ + Class to run module pytests. + + ... + + Attributes + ---------- + module_name : str + name of the tool to run tests for + no_prompts : bool + flat indicating if prompts are used + pytest_args : tuple + additional arguments passed to pytest command + + Methods + ------- + run(): + Run test steps + _check_inputs(): + Check inputs. Ask for module_name if not provided and check that the directory exists + _set_profile(): + Set software profile + _run_pytests(self): + Run pytest + """ + + def __init__( + self, + module_name=None, + no_prompts=False, + pytest_args="", + remote_url=None, + branch=None, + no_pull=False, + ): + self.module_name = module_name + self.no_prompts = no_prompts + self.pytest_args = pytest_args + + super().__init__(".", remote_url, branch, no_pull) + + def run(self): + """Run test steps""" + if not self.no_prompts: + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + ) + self._check_inputs() + self._set_profile() + self._check_profile() + self._run_pytests() + + def _check_inputs(self): + """Do more complex checks about supplied flags.""" + # Check modules directory structure + self.check_modules_structure() + + # Retrieving installed modules + if self.repo_type == "modules": + installed_modules = self.get_modules_clone_modules() + else: + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() + installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url) + + # Get the tool name if not specified + if self.module_name is None: + if self.no_prompts: + raise UserWarning( + "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." + ) + if not installed_modules: + raise UserWarning( + f"No installed modules were found from '{self.modules_repo.remote_url}'.\n" + f"Are you running the tests inside the nf-core/modules main directory?\n" + f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/" + ) + self.module_name = questionary.autocomplete( + "Tool name:", + choices=installed_modules, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Sanity check that the module directory exists + self._validate_folder_structure() + + def _validate_folder_structure(self): + """Validate that the modules follow the correct folder structure to run the tests: + - modules/nf-core/TOOL/SUBTOOL/ + - tests/modules/nf-core/TOOL/SUBTOOL/ + + """ + module_path = Path(self.default_modules_path) / self.module_name + test_path = Path(self.default_tests_path) / self.module_name + + if not (self.dir / module_path).is_dir(): + raise UserWarning( + f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" + ) + if not (self.dir / test_path).is_dir(): + raise UserWarning( + f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. " + "Are you running the tests inside the nf-core/modules main directory? " + "Do you have tests for the specified module?" + ) + + def _set_profile(self): + """Set $PROFILE env variable. + The config expects $PROFILE and Nextflow fails if it's not set. + """ + if os.environ.get("PROFILE") is None: + os.environ["PROFILE"] = "" + if self.no_prompts: + log.info( + "Setting environment variable '$PROFILE' to an empty string as not set.\n" + "Tests will run with Docker by default. " + "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." + ) + else: + question = { + "type": "list", + "name": "profile", + "message": "Choose software profile", + "choices": ["Docker", "Singularity", "Conda"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + profile = answer["profile"].lower() + os.environ["PROFILE"] = profile + log.info(f"Setting environment variable '$PROFILE' to '{profile}'") + + def _check_profile(self): + """Check if profile is available""" + profile = os.environ.get("PROFILE") + # Make sure the profile read from the environment is a valid Nextflow profile. + valid_nextflow_profiles = ["docker", "singularity", "conda"] + if profile in valid_nextflow_profiles: + if not which(profile): + raise UserWarning(f"Command '{profile}' not found - is it installed?") + else: + raise UserWarning( + f"The PROFILE '{profile}' set in the shell environment is not valid.\n" + f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." + ) + + def _run_pytests(self): + """Given a module name, run tests.""" + # Print nice divider line + console = rich.console.Console() + console.rule(self.module_name, style="black") + + # Set pytest arguments + command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] + command_args += self.pytest_args + + # Run pytest + log.info(f"Running pytest for module '{self.module_name}'") + sys.exit(pytest.main(command_args)) diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index cad349893e..a57bd9686c 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,3 +1,4 @@ from .create import SubworkflowCreate from .install import SubworkflowInstall +from .subworkflows_test import SubworkflowsTest from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 82f2b1441f..42536ffa45 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -30,8 +30,8 @@ class SubworkflowsTest(SubworkflowCommand): Attributes ---------- - module_name : str - name of the tool to run tests for + subworkflow_name : str + name of the subworkflow to run tests for no_prompts : bool flat indicating if prompts are used pytest_args : tuple @@ -42,7 +42,7 @@ class SubworkflowsTest(SubworkflowCommand): run(): Run test steps _check_inputs(): - Check inputs. Ask for module_name if not provided and check that the directory exists + Check inputs. Ask for subworkflow_name if not provided and check that the directory exists _set_profile(): Set software profile _run_pytests(self): @@ -77,7 +77,7 @@ def run(self): def _check_inputs(self): """Do more complex checks about supplied flags.""" - # Retrieving installed modules + # Retrieving installed subworkflows if self.repo_type == "modules": installed_subwf = self.get_components_clone_modules() else: @@ -85,8 +85,8 @@ def _check_inputs(self): modules_json.check_up_to_date() installed_subwf = modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - # Get the tool name if not specified - if self.module_name is None: + # Get the subworkflow name if not specified + if self.subworkflow_name is None: if self.no_prompts: raise UserWarning( "Subworkflow name not provided and prompts deactivated. Please provide the Subworkflow name SUBWORKFLOW." diff --git a/tests/subworkflows/subworkflows_test.py b/tests/subworkflows/subworkflows_test.py new file mode 100644 index 0000000000..adb0989b33 --- /dev/null +++ b/tests/subworkflows/subworkflows_test.py @@ -0,0 +1,41 @@ +"""Test the 'subworkflows test' command which runs module pytests.""" +import os +import shutil +from pathlib import Path + +import pytest + +import nf_core.subworkflows + +from ..utils import set_wd + + +def test_subworkflows_test_check_inputs(self): + """Test the check_inputs() function - raise UserWarning because module doesn't exist""" + with set_wd(self.nfcore_modules): + meta_builder = nf_core.subworkflows.SubworkflowsTest("none", True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() + assert "Cannot find directory" in str(excinfo.value) + + +def test_subworkflows_test_no_name_no_prompts(self): + """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided.""" + with set_wd(self.nfcore_modules): + meta_builder = nf_core.subworkflows.SubworkflowsTest(None, True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() + assert "Subworkflow name not provided and prompts deactivated." in str(excinfo.value) + + +def test_subworkflows_test_no_installed_subworkflows(self): + """Test the check_inputs() function - raise UserWarning because installed modules were not found""" + with set_wd(self.nfcore_modules): + module_dir = Path(self.nfcore_modules, "subworkflows") + shutil.rmtree(module_dir) + module_dir.mkdir() + meta_builder = nf_core.subworkflows.SubworkflowsTest(None, False, "") + meta_builder.repo_type = "modules" + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() + assert "No installed subworkflows were found" in str(excinfo.value) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index de52ed589c..7966283949 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -67,3 +67,9 @@ def setUp(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) + + from .subworkflows.subworkflows_test import ( + test_subworkflows_test_check_inputs, + test_subworkflows_test_no_installed_subworkflows, + test_subworkflows_test_no_name_no_prompts, + ) From 9d36b31602b6a89177e13ce00fa9ad8e232db2e0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 27 Oct 2022 16:46:55 +0200 Subject: [PATCH 390/854] fix isort --- nf_core/modules/__init__.py | 4 ++-- nf_core/modules/info.py | 2 +- nf_core/modules/lint/module_version.py | 2 +- nf_core/modules/modules_json.py | 2 +- nf_core/modules/modules_repo.py | 2 +- tests/test_modules.py | 10 +++++----- tests/test_subworkflows.py | 1 - 7 files changed, 11 insertions(+), 12 deletions(-) diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index c60c630fc0..060b39124b 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -4,10 +4,10 @@ from .install import ModuleInstall from .lint import ModuleLint from .list import ModuleList -from .modules_test import ModulesTest -from .modules_utils import ModuleException from .modules_json import ModulesJson from .modules_repo import ModulesRepo +from .modules_test import ModulesTest +from .modules_utils import ModuleException from .mulled import MulledImageNameGenerator from .patch import ModulePatch from .remove import ModuleRemove diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 635b8eba41..8fe5c75ead 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -13,9 +13,9 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson -from .modules_utils import get_repo_type from .modules_command import ModuleCommand from .modules_repo import NF_CORE_MODULES_REMOTE +from .modules_utils import get_repo_type log = logging.getLogger(__name__) diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index 52df63f06b..de77f31cf6 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -7,8 +7,8 @@ from pathlib import Path import nf_core -import nf_core.modules.modules_utils import nf_core.modules.modules_repo +import nf_core.modules.modules_utils log = logging.getLogger(__name__) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 615b35d98c..a674543124 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -11,8 +11,8 @@ import questionary from git.exc import GitCommandError -import nf_core.modules.modules_utils import nf_core.modules.modules_repo +import nf_core.modules.modules_utils import nf_core.utils from .modules_differ import ModulesDiffer diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 250446d012..22de2f9c37 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -10,8 +10,8 @@ import rich.progress from git.exc import GitCommandError, InvalidGitRepositoryError -import nf_core.modules.modules_utils import nf_core.modules.modules_json +import nf_core.modules.modules_utils from nf_core.utils import NFCORE_DIR log = logging.getLogger(__name__) diff --git a/tests/test_modules.py b/tests/test_modules.py index 9cbab99801..e1b9609699 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -152,11 +152,6 @@ def test_modulesrepo_class(self): test_modules_list_remote, test_modules_list_remote_gitlab, ) - from .modules.modules_test import ( - test_modules_test_check_inputs, - test_modules_test_no_installed_modules, - test_modules_test_no_name_no_prompts, - ) from .modules.modules_json import ( test_get_modules_json, test_mod_json_create, @@ -172,6 +167,11 @@ def test_modulesrepo_class(self): test_mod_json_with_empty_modules_value, test_mod_json_with_missing_modules_entry, ) + from .modules.modules_test import ( + test_modules_test_check_inputs, + test_modules_test_no_installed_modules, + test_modules_test_no_name_no_prompts, + ) from .modules.patch import ( test_create_patch_change, test_create_patch_no_change, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 7966283949..ea4fa986dd 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -67,7 +67,6 @@ def setUp(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) - from .subworkflows.subworkflows_test import ( test_subworkflows_test_check_inputs, test_subworkflows_test_no_installed_subworkflows, From 35ce5d230e419efe4743f84055612fd79371f97a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 27 Oct 2022 17:11:31 +0200 Subject: [PATCH 391/854] add components_install --- nf_core/components/components_install.py | 127 +++++++++++++++++++++++ nf_core/modules/install.py | 94 ++++------------- nf_core/modules/lint/module_version.py | 2 +- nf_core/modules/module_utils.py | 2 +- nf_core/modules/modules_json.py | 4 +- nf_core/modules/modules_repo.py | 61 +++-------- nf_core/modules/update.py | 2 +- nf_core/subworkflows/install.py | 122 ++++++++-------------- tests/modules/update.py | 4 +- 9 files changed, 214 insertions(+), 204 deletions(-) create mode 100644 nf_core/components/components_install.py diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py new file mode 100644 index 0000000000..2925257922 --- /dev/null +++ b/nf_core/components/components_install.py @@ -0,0 +1,127 @@ +import glob +import json +import logging +import os +import re + +import jinja2 +import questionary +import rich + +import nf_core.modules.module_utils +import nf_core.utils +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME + +log = logging.getLogger(__name__) + + +def verify_sha(modules_repo, prompt, sha): + """ + Verify that 'sha' and 'prompt' arguments are not provided together. + Verify that the provided SHA exists in the repo. + + Arguments: + modules_repo (ModulesRepo): ModulesRepo object + prompt (bool): prompt asking for SHA + sha (str): provided sha + """ + if prompt and sha is not None: + log.error("Cannot use '--sha' and '--prompt' at the same time!") + return False + + if sha: + if not modules_repo.sha_exists_on_branch(sha): + log.error(f"Commit SHA '{sha}' doesn't exist in '{modules_repo.remote_url}'") + return False + + return True + + +def collect_and_verify_name(component_type, component, modules_repo): + """ + Collect component name. + Check that the supplied name is an available module/subworkflow. + """ + if component is None: + component = questionary.autocomplete( + f"{'Tool' if component_type == 'modules' else 'Subworkflow'} name:", + choices=modules_repo.get_avail_components(component_type), + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Check that the supplied name is an available module/subworkflow + if component and component not in modules_repo.get_avail_components(component_type): + log.error(f"{component_type[:-1].title()} '{component}' not found in list of available {component_type}.") + log.info(f"Use the command 'nf-core {component_type} list' to view available software") + return False + + if not modules_repo.component_exists(component, component_type): + warn_msg = f"{component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" + log.warning(warn_msg) + return False + + return component + + +def check_component_installed(component_type, component, current_version, component_dir, modules_repo, force): + """ + Check that the module/subworkflow is not already installed + """ + if (current_version is not None and os.path.exists(component_dir)) and not force: + log.info(f"{component_type[:-1].title()} is already installed.") + + message = "?" if component_type == "modules" else " of this subworkflow and all it's imported modules?" + force = questionary.confirm( + f"{component_type[:-1].title()} {component} is already installed. \nDo you want to force the reinstallation{message}", + style=nf_core.utils.nfcore_question_style, + default=False, + ).unsafe_ask() + + if not force: + repo_flag = "" if modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {modules_repo.remote_url} " + branch_flag = "" if modules_repo.branch == "master" else f"-b {modules_repo.branch} " + + log.info( + f"To update '{component}' run 'nf-core {component_type} {repo_flag}{branch_flag}update {component}'. To force reinstallation use '--force'" + ) + return False + + return True + + +def get_version(component, component_type, sha, prompt, current_version, modules_repo): + """ + Get the version to install + """ + if sha: + version = sha + elif prompt: + try: + version = nf_core.modules.module_utils.prompt_module_version_sha( + component, + installed_sha=current_version, + modules_repo=modules_repo, + ) + except SystemError as e: + log.error(e) + return False + else: + # Fetch the latest commit for the module + version = modules_repo.get_latest_component_version(component, component_type) + return version + + +def clean_modules_json(component, component_type, modules_repo, modules_json): + """ + Remove installed version of module/subworkflow + """ + for repo_url, repo_content in modules_json.modules_json["repos"].items(): + for dir, dir_components in repo_content[component_type].items(): + for name, _ in dir_components.items(): + if name == component and dir == modules_repo.repo_path: + repo_to_remove = repo_url + log.info( + f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" + ) + modules_json.remove_entry(component, repo_to_remove, modules_repo.repo_path) + break diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index a787ac8077..5c7bda36a6 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -3,6 +3,7 @@ import questionary +import nf_core.components.components_install import nf_core.modules.module_utils import nf_core.utils from nf_core.modules.modules_json import ModulesJson @@ -44,36 +45,18 @@ def install(self, module, silent=False): modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - if self.prompt and self.sha is not None: - log.error("Cannot use '--sha' and '--prompt' at the same time!") + # Verify SHA + if not nf_core.components.components_install.verify_sha(self.modules_repo, self.prompt, self.sha): return False - # Verify that the provided SHA exists in the repo - if self.sha: - if not self.modules_repo.sha_exists_on_branch(self.sha): - log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") - return False - - if module is None: - module = questionary.autocomplete( - "Tool name:", - choices=self.modules_repo.get_avail_components(self.component_type), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Check that the supplied name is an available module - if module and module not in self.modules_repo.get_avail_components(self.component_type): - log.error(f"Module '{module}' not found in list of available modules.") - log.info("Use the command 'nf-core modules list' to view available software") - return False - - if not self.modules_repo.component_exists(module, self.component_type): - warn_msg = ( - f"Module '{module}' not found in remote '{self.modules_repo.remote_url}' ({self.modules_repo.branch})" - ) - log.warning(warn_msg) + # Check and verify module name + module = nf_core.components.components_install.collect_and_verify_name( + self.component_type, module, self.modules_repo + ) + if not module: return False + # Get current version current_version = modules_json.get_module_version( module, self.modules_repo.remote_url, self.modules_repo.repo_path ) @@ -85,55 +68,24 @@ def install(self, module, silent=False): module_dir = os.path.join(install_folder, module) # Check that the module is not already installed - if (current_version is not None and os.path.exists(module_dir)) and not self.force: - log.info("Module is already installed.") - - self.force = questionary.confirm( - f"Module {module} is already installed. Do you want to force the reinstallation?", - style=nf_core.utils.nfcore_question_style, - default=False, - ).unsafe_ask() - - if not self.force: - repo_flag = ( - "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " - ) - branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " - - log.info( - f"To update '{module}' run 'nf-core modules {repo_flag}{branch_flag}update {module}'. To force reinstallation use '--force'" - ) - return False - - if self.sha: - version = self.sha - elif self.prompt: - try: - version = nf_core.modules.module_utils.prompt_module_version_sha( - module, - installed_sha=current_version, - modules_repo=self.modules_repo, - ) - except SystemError as e: - log.error(e) - return False - else: - # Fetch the latest commit for the module - version = self.modules_repo.get_latest_module_version(module) + if not nf_core.components.components_install.check_component_installed( + self.component_type, module, current_version, module_dir, self.modules_repo, self.force + ): + return False + version = nf_core.components.components_install.get_version( + module, self.component_type, self.sha, self.prompt, current_version, self.modules_repo + ) + if not version: + return False + + # Remove module if force is set if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") self.clear_component_dir(module, module_dir) - for repo_url, repo_content in modules_json.modules_json["repos"].items(): - for dir, dir_modules in repo_content["modules"].items(): - for name, _ in dir_modules.items(): - if name == module and dir == self.modules_repo.repo_path: - repo_to_remove = repo_url - log.info( - f"Removing module '{self.modules_repo.repo_path}/{module}' from repo '{repo_to_remove}' from modules.json" - ) - modules_json.remove_entry(module, repo_to_remove, self.modules_repo.repo_path) - break + nf_core.components.components_install.clean_modules_json( + module, self.component_type, self.modules_repo, modules_json + ) log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'") log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}") diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index 9077cc5371..441930b60a 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -38,7 +38,7 @@ def module_version(module_lint_object, module): # Check whether a new version is available try: modules_repo = nf_core.modules.modules_repo.ModulesRepo() - module_git_log = modules_repo.get_module_git_log(module.module_name) + module_git_log = modules_repo.get_component_git_log(module.module_name, "modules") if version == next(module_git_log)["git_sha"]: module.passed.append(("module_version", "Module is the latest version", module.module_dir)) else: diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 49b58bec64..8231320e27 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -201,7 +201,7 @@ def prompt_module_version_sha(module, modules_repo, installed_sha=None): git_sha = "" page_nbr = 1 - all_commits = modules_repo.get_module_git_log(module) + all_commits = modules_repo.get_component_git_log(module, "modules") next_page_commits = [next(all_commits, None) for _ in range(10)] next_page_commits = [commit for commit in next_page_commits if commit is not None] diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index a095049e65..4f2175b2d0 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -300,7 +300,9 @@ def find_correct_commit_sha(self, module_name, module_path, modules_repo): # Find the correct commit SHA for the local module files. # We iterate over the commit history for the module until we find # a revision that matches the file contents - commit_shas = (commit["git_sha"] for commit in modules_repo.get_module_git_log(module_name, depth=1000)) + commit_shas = ( + commit["git_sha"] for commit in modules_repo.get_component_git_log(module_name, "modules", depth=1000) + ) for commit_sha in commit_shas: if all(modules_repo.module_files_identical(module_name, module_path, commit_sha).values()): return commit_sha diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 5621a02249..b17a6c04c0 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -363,70 +363,39 @@ def module_files_identical(self, module_name, base_path, commit): self.checkout_branch() return files_identical - def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00Z"): + def get_component_git_log(self, component_name, component_type, depth=None): """ - Fetches the commit history the of requested module since a given date. The default value is + Fetches the commit history the of requested module/subworkflow since a given date. The default value is not arbitrary - it is the last time the structure of the nf-core/modules repository was had an update breaking backwards compatibility. Args: - module_name (str): Name of module + component_name (str): Name of module/subworkflow modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question - per_page (int): Number of commits per page returned by API - page_nbr (int): Page number of the retrieved commits - since (str): Only show commits later than this timestamp. - Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. Returns: ( dict ): Iterator of commit SHAs and associated (truncated) message """ self.checkout_branch() - module_path = os.path.join("modules", self.repo_path, module_name) - commits_new = self.repo.iter_commits(max_count=depth, paths=module_path) + component_path = os.path.join(component_type, self.repo_path, component_name) + commits_new = self.repo.iter_commits(max_count=depth, paths=component_path) commits_new = [ {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_new ] - # Grab commits also from previous modules structure - module_path = os.path.join("modules", module_name) - commits_old = self.repo.iter_commits(max_count=depth, paths=module_path) - commits_old = [ - {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_old - ] - commits = iter(commits_new + commits_old) - return commits - - def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): - """ - Fetches the commit history the of requested subworkflow since a given date. The default value is - not arbitrary - it is the last time the structure of the nf-core/subworkflow repository was had an - update breaking backwards compatibility. - Args: - subworkflow_name (str): Name of subworkflow - modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question - per_page (int): Number of commits per page returned by API - page_nbr (int): Page number of the retrieved commits - since (str): Only show commits later than this timestamp. - Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. - - Returns: - ( dict ): Iterator of commit SHAs and associated (truncated) message - """ - self.checkout_branch() - subworkflow_path = os.path.join("subworkflows", self.repo_path, subworkflow_name) - commits = self.repo.iter_commits(max_count=depth, paths=subworkflow_path) - commits = ({"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits) + if component_type == "modules": + # Grab commits also from previous modules structure + component_path = os.path.join("modules", component_name) + commits_old = self.repo.iter_commits(max_count=depth, paths=component_path) + commits_old = [ + {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_old + ] + commits = iter(commits_new + commits_old) return commits - def get_latest_module_version(self, module_name): - """ - Returns the latest commit in the repository - """ - return list(self.get_module_git_log(module_name, depth=1))[0]["git_sha"] - - def get_latest_subworkflow_version(self, module_name): + def get_latest_component_version(self, component_name, component_type): """ Returns the latest commit in the repository """ - return list(self.get_subworkflow_git_log(module_name, depth=1))[0]["git_sha"] + return list(self.get_component_git_log(component_name, component_type, depth=1))[0]["git_sha"] def sha_exists_on_branch(self, sha): """ diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index ce39b4b765..db7832659d 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -155,7 +155,7 @@ def update(self, module=None): module, modules_repo=modules_repo, installed_sha=current_version ) else: - version = modules_repo.get_latest_module_version(module) + version = modules_repo.get_latest_component_version(module, "modules") if current_version is not None and not self.force: if current_version == version: diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index f0959e92d9..1a84d8c45c 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -5,6 +5,7 @@ import questionary +import nf_core.components.components_install import nf_core.modules.module_utils import nf_core.utils from nf_core.modules.install import ModuleInstall @@ -44,34 +45,18 @@ def install(self, subworkflow, silent=False): modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - if self.prompt and self.sha is not None: - log.error("Cannot use '--sha' and '--prompt' at the same time!") + # Verify SHA + if not nf_core.components.components_install.verify_sha(self.modules_repo, self.prompt, self.sha): return False - # Verify that the provided SHA exists in the repo - if self.sha: - if not self.modules_repo.sha_exists_on_branch(self.sha): - log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") - return False - - if subworkflow is None: - subworkflow = questionary.autocomplete( - "Subworkflow name:", - choices=self.modules_repo.get_avail_components(self.component_type), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Check that the supplied name is an available subworkflow - if subworkflow and subworkflow not in self.modules_repo.get_avail_components(self.component_type): - log.error(f"Subworkflow '{subworkflow}' not found in list of available subworkflows.") - log.info("Use the command 'nf-core subworkflows list' to view available software") - return False - - if not self.modules_repo.component_exists(subworkflow, self.component_type): - warn_msg = f"Subworkflow '{subworkflow}' not found in remote '{self.modules_repo.remote_url}' ({self.modules_repo.branch})" - log.warning(warn_msg) + # Check and verify subworkflow name + subworkflow = nf_core.components.components_install.collect_and_verify_name( + self.component_type, subworkflow, self.modules_repo + ) + if not subworkflow: return False + # Get current version current_version = modules_json.get_subworkflow_version( subworkflow, self.modules_repo.remote_url, self.modules_repo.repo_path ) @@ -83,55 +68,24 @@ def install(self, subworkflow, silent=False): subworkflow_dir = os.path.join(install_folder, subworkflow) # Check that the subworkflow is not already installed - if (current_version is not None and os.path.exists(subworkflow_dir)) and not self.force: - log.info("Subworkflow is already installed.") - - self.force = questionary.confirm( - f"Subworkflow {subworkflow} is already installed.\nDo you want to force the reinstallation of this subworkflow and all it's imported modules?", - style=nf_core.utils.nfcore_question_style, - default=False, - ).unsafe_ask() - - if not self.force: - repo_flag = ( - "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} " - ) - branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " + if not nf_core.components.components_install.check_component_installed( + self.component_type, subworkflow, current_version, subworkflow_dir, self.modules_repo, self.force + ): + return False - log.info( - f"To update '{subworkflow}' run 'nf-core subworkflow {repo_flag}{branch_flag}update {subworkflow}'. To force reinstallation use '--force'" - ) - return False - - if self.sha: - version = self.sha - elif self.prompt: - try: - version = nf_core.modules.module_utils.prompt_module_version_sha( - subworkflow, - installed_sha=current_version, - modules_repo=self.modules_repo, - ) - except SystemError as e: - log.error(e) - return False - else: - # Fetch the latest commit for the subworkflow - version = self.modules_repo.get_latest_subworkflow_version(subworkflow) + version = nf_core.components.components_install.get_version( + subworkflow, self.component_type, self.sha, self.prompt, current_version, self.modules_repo + ) + if not version: + return False + # Remove subworkflow if force is set if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") self.clear_component_dir(subworkflow, subworkflow_dir) - for repo_url, repo_content in modules_json.modules_json["repos"].items(): - for dir, dir_subworkflow in repo_content["subworkflows"].items(): - for name, _ in dir_subworkflow.items(): - if name == subworkflow and dir == self.modules_repo.repo_path: - repo_to_remove = repo_url - log.info( - f"Removing subworkflow '{self.modules_repo.repo_path}/{subworkflow}' from repo '{repo_to_remove}' from modules.json" - ) - modules_json.remove_entry(subworkflow, repo_to_remove, self.modules_repo.repo_path) - break + nf_core.components.components_install.clean_modules_json( + subworkflow, self.component_type, self.modules_repo, modules_json + ) log.info(f"{'Rei' if self.force else 'I'}nstalling '{subworkflow}'") log.debug(f"Installing subworkflow '{subworkflow}' at hash {version} from {self.modules_repo.remote_url}") @@ -141,19 +95,7 @@ def install(self, subworkflow, silent=False): return False # Install included modules and subworkflows - modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) - for s_install in subworkflows_to_install: - self.install(s_install, silent=True) - for m_install in modules_to_install: - module_install = ModuleInstall( - self.dir, - force=self.force, - prompt=self.prompt, - sha=self.sha, - remote_url=self.modules_repo.remote_url, - branch=self.modules_repo.branch, - ) - module_install.install(m_install, silent=True) + self.install_included_components(subworkflow_dir) if not silent: # Print include statement @@ -190,3 +132,21 @@ def get_modules_subworkflows_to_install(self, subworkflow_dir): elif link.startswith("../"): subworkflows.append(name.lower()) return modules, subworkflows + + def install_included_components(self, subworkflow_dir): + """ + Install included modules and subworkflows + """ + modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) + for s_install in subworkflows_to_install: + self.install(s_install, silent=True) + for m_install in modules_to_install: + module_install = ModuleInstall( + self.dir, + force=self.force, + prompt=self.prompt, + sha=self.sha, + remote_url=self.modules_repo.remote_url, + branch=self.modules_repo.branch, + ) + module_install.install(m_install, silent=True) diff --git a/tests/modules/update.py b/tests/modules/update.py index bb1da50f52..9ff098d88e 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -56,7 +56,7 @@ def test_install_at_hash_and_update(self): mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() # Get the up-to-date git_sha for the module from the ModulesRepo object - correct_git_sha = update_obj.modules_repo.get_latest_module_version("trimgalore") + correct_git_sha = update_obj.modules_repo.get_latest_component_version("trimgalore", "modules") current_git_sha = mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] assert correct_git_sha == current_git_sha @@ -96,7 +96,7 @@ def test_update_all(self): mod_json = mod_json_obj.get_modules_json() # Loop through all modules and check that they are updated (according to the modules.json file) for mod in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: - correct_git_sha = list(update_obj.modules_repo.get_module_git_log(mod, depth=1))[0]["git_sha"] + correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] assert correct_git_sha == current_git_sha From 51bafdef5d83608a4f0b643d4455c33125b3eb40 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Thu, 27 Oct 2022 10:35:47 -0500 Subject: [PATCH 392/854] style: mullled => mulled --- tests/{test_mullled.py => test_mulled.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_mullled.py => test_mulled.py} (100%) diff --git a/tests/test_mullled.py b/tests/test_mulled.py similarity index 100% rename from tests/test_mullled.py rename to tests/test_mulled.py From dc4e4d6634f456d612ca059ae7076ac1dd6ee6e1 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Thu, 27 Oct 2022 16:29:53 -0500 Subject: [PATCH 393/854] Add nextflow extension pack. --- nf_core/pipeline-template/.devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.devcontainer/devcontainer.json b/nf_core/pipeline-template/.devcontainer/devcontainer.json index c82f6703a5..f8df1d2be2 100644 --- a/nf_core/pipeline-template/.devcontainer/devcontainer.json +++ b/nf_core/pipeline-template/.devcontainer/devcontainer.json @@ -30,7 +30,8 @@ // Add the IDs of extensions you want installed when the container is created. "extensions": [ "ms-python.python", - "ms-python.vscode-pylance" + "ms-python.vscode-pylance", + "nf-core.nf-core-extensionpack" ] } }, From e8f0502ea100ae4a16eb5a6b04476da830fea401 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Thu, 27 Oct 2022 16:35:38 -0500 Subject: [PATCH 394/854] Update syntax. --- nf_core/pipeline-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 8eceb4816f..364fde0852 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -80,7 +80,7 @@ Devcontainer specs: - [DevContainer config](.devcontainer/devcontainer.json) - [Dockerfile](.devcontainer/Dockerfile) -# Getting started +# To get started: - Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) - Tools installed - nf-core From d8bd51742fcf05f135a60b285b2b0af2e280172d Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Thu, 27 Oct 2022 16:40:27 -0500 Subject: [PATCH 395/854] Moved section to root Contributing directory. --- .github/CONTRIBUTING.md | 13 +++++++++++++ nf_core/pipeline-template/README.md | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 800ba7ab10..851d88cbcc 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -127,3 +127,16 @@ You can replicate this process locally with the following commands: nf-core create -n testpipeline -d "This pipeline is for testing" nf-core lint nf-core-testpipeline ``` + +## Codespaces +This repo includes a devcontainer configuration which will create a GitHub Codespace for Nextflow development! + +Devcontainer specs: +- [DevContainer config](.devcontainer/devcontainer.json) +- [Dockerfile](.devcontainer/Dockerfile) + +# To get started: +- Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) +- Tools installed + - nf-core + - nextflow diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 364fde0852..02a32f1f6e 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -73,19 +73,6 @@ The results obtained from the full-sized test can be viewed on the [nf-core webs {% if branded -%} -## Codespaces -This repo includes a devcontainer configuration which will create a GitHub Codespace for Nextflow development! - -Devcontainer specs: -- [DevContainer config](.devcontainer/devcontainer.json) -- [Dockerfile](.devcontainer/Dockerfile) - -# To get started: -- Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) -- Tools installed - - nf-core - - nextflow - ## Documentation The {{ name }} pipeline comes with documentation about the pipeline [usage](https://nf-co.re/{{ short_name }}/usage), [parameters](https://nf-co.re/{{ short_name }}/parameters) and [output](https://nf-co.re/{{ short_name }}/output). From 3817949d26fe00764d7d559fe6a1b097f0e02404 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Thu, 27 Oct 2022 16:44:17 -0500 Subject: [PATCH 396/854] Make more parity with Gitpod. --- nf_core/pipeline-template/.devcontainer/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nf_core/pipeline-template/.devcontainer/Dockerfile b/nf_core/pipeline-template/.devcontainer/Dockerfile index 85ad388646..8e4c9e259f 100644 --- a/nf_core/pipeline-template/.devcontainer/Dockerfile +++ b/nf_core/pipeline-template/.devcontainer/Dockerfile @@ -18,4 +18,16 @@ RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bi RUN conda install -c bioconda nextflow RUN pip install nf-core +RUN conda update -n base -c defaults conda && \ + conda config --add channels defaults && \ + conda config --add channels bioconda && \ + conda config --add channels conda-forge && \ + conda install \ + pytest-workflow=1.6.0 \ + mamba=0.24.0 \ + pip=22.1.2 \ + black=22.6.0 \ + prettier=2.7.1 \ + -n base && \ + conda clean --all -f -y From 862fa6b555afdabc955b8fcba74811c970fa2739 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Thu, 27 Oct 2022 16:45:10 -0500 Subject: [PATCH 397/854] Make more parity with Gitpod. --- nf_core/pipeline-template/.devcontainer/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.devcontainer/Dockerfile b/nf_core/pipeline-template/.devcontainer/Dockerfile index 8e4c9e259f..c866df8c51 100644 --- a/nf_core/pipeline-template/.devcontainer/Dockerfile +++ b/nf_core/pipeline-template/.devcontainer/Dockerfile @@ -16,13 +16,14 @@ RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bi # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends -RUN conda install -c bioconda nextflow RUN pip install nf-core RUN conda update -n base -c defaults conda && \ conda config --add channels defaults && \ conda config --add channels bioconda && \ conda config --add channels conda-forge && \ conda install \ + openjdk=11.0.15 \ + nextflow=22.04.0 \ pytest-workflow=1.6.0 \ mamba=0.24.0 \ pip=22.1.2 \ From cc144b3869aae306a29c8bd674fa65d6bdcd1689 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Thu, 27 Oct 2022 16:45:30 -0500 Subject: [PATCH 398/854] Add a codspaces for the tools repo. --- .devcontainer/Dockerfile | 34 ++++++++++++++++++++ .devcontainer/add-notice.sh | 20 ++++++++++++ .devcontainer/devcontainer.json | 56 +++++++++++++++++++++++++++++++++ .devcontainer/noop.txt | 3 ++ 4 files changed, 113 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/add-notice.sh create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/noop.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..c866df8c51 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,34 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda/.devcontainer/base.Dockerfile + +FROM mcr.microsoft.com/vscode/devcontainers/anaconda:0-3 + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="none" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# Copy environment.yml (if found) to a temp location so we update the environment. Also +# copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists. +COPY environment.yml* .devcontainer/noop.txt /tmp/conda-tmp/ +RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; fi \ + && rm -rf /tmp/conda-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +RUN pip install nf-core +RUN conda update -n base -c defaults conda && \ + conda config --add channels defaults && \ + conda config --add channels bioconda && \ + conda config --add channels conda-forge && \ + conda install \ + openjdk=11.0.15 \ + nextflow=22.04.0 \ + pytest-workflow=1.6.0 \ + mamba=0.24.0 \ + pip=22.1.2 \ + black=22.6.0 \ + prettier=2.7.1 \ + -n base && \ + conda clean --all -f -y + diff --git a/.devcontainer/add-notice.sh b/.devcontainer/add-notice.sh new file mode 100644 index 0000000000..0417abb6cd --- /dev/null +++ b/.devcontainer/add-notice.sh @@ -0,0 +1,20 @@ +# Display a notice when not running in GitHub Codespaces + +cat << 'EOF' > /usr/local/etc/vscode-dev-containers/conda-notice.txt +When using "conda" from outside of GitHub Codespaces, note the Anaconda repository +contains restrictions on commercial use that may impact certain organizations. See +https://aka.ms/vscode-remote/conda/anaconda + +EOF + +notice_script="$(cat << 'EOF' +if [ -t 1 ] && [ "${IGNORE_NOTICE}" != "true" ] && [ "${TERM_PROGRAM}" = "vscode" ] && [ "${CODESPACES}" != "true" ] && [ ! -f "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed" ]; then + cat "/usr/local/etc/vscode-dev-containers/conda-notice.txt" + mkdir -p "$HOME/.config/vscode-dev-containers" + ((sleep 10s; touch "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed") &) +fi +EOF +)" + +echo "${notice_script}" | tee -a /etc/bash.bashrc >> /etc/zsh/zshrc + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..f8df1d2be2 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,56 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda +{ + "name": "Anaconda (Python 3)", + "build": { + "context": "..", + "dockerfile": "Dockerfile", + "args": { + "NODE_VERSION": "lts/*" + } + }, + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/opt/conda/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", + "python.formatting.yapfPath": "/opt/conda/bin/yapf", + "python.linting.flake8Path": "/opt/conda/bin/flake8", + "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", + "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", + "python.linting.pylintPath": "/opt/conda/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "nf-core.nf-core-extensionpack" + ] + } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "python --version", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + "features": { + "docker-in-docker": "20.10", + "git": "os-provided", + "ghcr.io/devcontainers/features/conda:1": {}, + "ghcr.io/devcontainers/features/docker-from-docker:1": {}, + "ghcr.io/devcontainers/features/git:1": {} + } +} + + diff --git a/.devcontainer/noop.txt b/.devcontainer/noop.txt new file mode 100644 index 0000000000..09c5ae9b86 --- /dev/null +++ b/.devcontainer/noop.txt @@ -0,0 +1,3 @@ +This file copied into the container along with environment.yml* from the parent +folder. This file is included to prevents the Dockerfile COPY instruction from +failing if no environment.yml is found. From c66fab45a030808ff997a0b62df46ee5c9035c46 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 28 Oct 2022 09:10:07 +0200 Subject: [PATCH 399/854] modify modules_utils name --- nf_core/components/components_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 2925257922..99833b54e3 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -8,7 +8,7 @@ import questionary import rich -import nf_core.modules.module_utils +import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME @@ -97,7 +97,7 @@ def get_version(component, component_type, sha, prompt, current_version, modules version = sha elif prompt: try: - version = nf_core.modules.module_utils.prompt_module_version_sha( + version = nf_core.modules.modules_utils.prompt_module_version_sha( component, installed_sha=current_version, modules_repo=modules_repo, From d983f365f63105f85a1c55076791acf35e82eb66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jennifer=20M=C3=BCller?= <43847598+jenmuell@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:04:52 +0200 Subject: [PATCH 400/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4b454d186f..ae36920036 100644 --- a/README.md +++ b/README.md @@ -540,9 +540,9 @@ There are four flags that you can use with this command: - `--web-only`: Skips comparison of the schema against the pipeline parameters and only launches the web tool. - `--url `: Supply a custom URL for the online tool. Useful when testing locally. -### Display the content of a pipeline schema +### Display the documentation for a pipeline schema -To get an impression about the current pipeline schema you can display the content of the `nextflow_schema.json` with `nf-core schema docs `. This will print a table with the content of your schema in the standard output. +To get an impression about the current pipeline schema you can display the content of the `nextflow_schema.json` with `nf-core schema docs `. This will print the content of your schema in Markdown format to the standard output. There are four flags that you can use with this command: @@ -553,13 +553,13 @@ There are four flags that you can use with this command: ### Add new parameters to the pipeline schema -If you want to add an parameter to the schema, you first have to add the parameter and its default value to the `nextflow.config` file in the `params` block. Afterwards, you run the command `nf-core schema build` to add the parameters to your schema and open the graphical interface to easily modify the schema. +If you want to add a parameter to the schema, you first have to add the parameter and its default value to the `nextflow.config` file with the `params` scope. Afterwards, you run the command `nf-core schema build` to add the parameters to your schema and open the graphical interface to easily modify the schema. -The graphical interface is oganzised in groups and within the groups the single parameters are stored. For an better overview you can collapse all groups with the `Collapse groups` button, than your new parameters will be the only remaining one at the bottom of the page. Now you can either create a new group with the `Add group` button or drag and drop the paramters in an existing group. Therefor the group has to be expanded. The group title will be displayed, if you run your pipeline with the `--help` flag and its description apears on the parameter page of your pipeline. +The graphical interface is oganzised in groups and within the groups the single parameters are stored. For a better overview you can collapse all groups with the `Collapse groups` button, then your new parameters will be the only remaining one at the bottom of the page. Now you can either create a new group with the `Add group` button or drag and drop the paramters in an existing group. Therefor the group has to be expanded. The group title will be displayed, if you run your pipeline with the `--help` flag and its description apears on the parameter page of your pipeline. -Now you can start to change the parameter itself. The description is a short explanation about the parameter, that apears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. If you want specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For intergers you can define a min and max value, and for strings the file extension can be specified. +Now you can start to change the parameter itself. The description is a short explanation about the parameter, that apears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For intergers you can define a min and max value, and for strings the file extension can be specified. -After you filled your schema, click on the `Finished` button in the top rigth corner, this will automatically update your `nextflow_schema.json`. If this is not working you can copy the schema from the graphical interface and paste it in your `nextflow_schema.json`. +After you filled your schema, click on the `Finished` button in the top rigth corner, this will automatically update your `nextflow_schema.json`. If this is not working you can copy the schema from the graphical interface and paste it in your `nextflow_schema.json` file. ### Update existing pipeline schema From ead27ddfd3d0083af4e26515bf9879fa10cdf677 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 28 Oct 2022 14:31:54 +0000 Subject: [PATCH 401/854] [automated] Fix code linting --- README.md | 20 ++++++++++---------- nf_core/pipeline-template/docs/usage.md | 12 ++++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ae36920036..64c34e4ae6 100644 --- a/README.md +++ b/README.md @@ -542,28 +542,28 @@ There are four flags that you can use with this command: ### Display the documentation for a pipeline schema -To get an impression about the current pipeline schema you can display the content of the `nextflow_schema.json` with `nf-core schema docs `. This will print the content of your schema in Markdown format to the standard output. +To get an impression about the current pipeline schema you can display the content of the `nextflow_schema.json` with `nf-core schema docs `. This will print the content of your schema in Markdown format to the standard output. There are four flags that you can use with this command: -- `--output `: Output filename. Defaults to standard out. -- `--format [markdown|html]`: Format to output docs in. -- `--force`: Overwrite existing files -- `--columns `: CSV list of columns to include in the parameter tables +- `--output `: Output filename. Defaults to standard out. +- `--format [markdown|html]`: Format to output docs in. +- `--force`: Overwrite existing files +- `--columns `: CSV list of columns to include in the parameter tables ### Add new parameters to the pipeline schema -If you want to add a parameter to the schema, you first have to add the parameter and its default value to the `nextflow.config` file with the `params` scope. Afterwards, you run the command `nf-core schema build` to add the parameters to your schema and open the graphical interface to easily modify the schema. +If you want to add a parameter to the schema, you first have to add the parameter and its default value to the `nextflow.config` file with the `params` scope. Afterwards, you run the command `nf-core schema build` to add the parameters to your schema and open the graphical interface to easily modify the schema. -The graphical interface is oganzised in groups and within the groups the single parameters are stored. For a better overview you can collapse all groups with the `Collapse groups` button, then your new parameters will be the only remaining one at the bottom of the page. Now you can either create a new group with the `Add group` button or drag and drop the paramters in an existing group. Therefor the group has to be expanded. The group title will be displayed, if you run your pipeline with the `--help` flag and its description apears on the parameter page of your pipeline. +The graphical interface is oganzised in groups and within the groups the single parameters are stored. For a better overview you can collapse all groups with the `Collapse groups` button, then your new parameters will be the only remaining one at the bottom of the page. Now you can either create a new group with the `Add group` button or drag and drop the paramters in an existing group. Therefor the group has to be expanded. The group title will be displayed, if you run your pipeline with the `--help` flag and its description apears on the parameter page of your pipeline. -Now you can start to change the parameter itself. The description is a short explanation about the parameter, that apears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For intergers you can define a min and max value, and for strings the file extension can be specified. +Now you can start to change the parameter itself. The description is a short explanation about the parameter, that apears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For intergers you can define a min and max value, and for strings the file extension can be specified. -After you filled your schema, click on the `Finished` button in the top rigth corner, this will automatically update your `nextflow_schema.json`. If this is not working you can copy the schema from the graphical interface and paste it in your `nextflow_schema.json` file. +After you filled your schema, click on the `Finished` button in the top rigth corner, this will automatically update your `nextflow_schema.json`. If this is not working you can copy the schema from the graphical interface and paste it in your `nextflow_schema.json` file. ### Update existing pipeline schema -Important for the update of a pipeline schema is, that if you want to change the default value of a parameter, you should change it in the `nextflow.config` file, since the value in the config file overwrites the value in the pipeline schema. To change any other parameter use `nf-core schema build --web-only` to open the graphical interface without rebuilding the pipeline schema. Now, you can change your parameters as mentioned above but keep in mind that changing the parameter datatype is depending on the default value you specified in the `nextflow.config` file. +Important for the update of a pipeline schema is, that if you want to change the default value of a parameter, you should change it in the `nextflow.config` file, since the value in the config file overwrites the value in the pipeline schema. To change any other parameter use `nf-core schema build --web-only` to open the graphical interface without rebuilding the pipeline schema. Now, you can change your parameters as mentioned above but keep in mind that changing the parameter datatype is depending on the default value you specified in the `nextflow.config` file. ### Linting a pipeline schema diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index afaceadbfe..843298a3d9 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -33,7 +33,7 @@ CONTROL_REP1,AEG588A1_S1_L004_R1_001.fastq.gz,AEG588A1_S1_L004_R2_001.fastq.gz ### Full samplesheet -The pipeline will auto-detect whether a sample is single- or paired-end using the information provided in the samplesheet. The samplesheet can have as many columns as you desire, however, there is a strict requirement for the first 3 columns to match those defined in the table below. +The pipeline will auto-detect whether a sample is single- or paired-end using the information provided in the samplesheet. The samplesheet can have as many columns as you desire, however, there is a strict requirement for the first 3 columns to match those defined in the table below. A final samplesheet file consisting of both single- and paired-end data may look something like the one below. This is for 6 samples, where `TREATMENT_REP3` has been sequenced twice. @@ -87,7 +87,7 @@ nextflow pull {{ name }} It is a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. -First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest pipeline version - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. +First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest pipeline version - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. @@ -99,7 +99,7 @@ This version number will be logged in reports when you run the pipeline, so that Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -111,7 +111,7 @@ The pipeline also dynamically loads configurations from [https://github.com/nf-c Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. -If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended, since it can lead to different results on different machines dependent on the computer enviroment. +If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended, since it can lead to different results on different machines dependent on the computer enviroment. - `test` - A profile with a complete configuration for automated testing @@ -129,7 +129,6 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. - ### `-resume` Specify this when restarting a pipeline. Nextflow will use cached results from any pipeline steps where the inputs are the same, continuing from where it got to previously. For input to be considered the same, not only the names must be identical but the files' contents as well. For more info about this parameter, see [this blog post](https://www.nextflow.io/blog/2019/demystifying-nextflow-resume.html). @@ -176,9 +175,10 @@ Work dir: Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` ``` + #### For beginners -A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. +A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. #### Advanced option on process level From 4799af18b9a1b2ff56467dd3dd27eb99334a3951 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 28 Oct 2022 09:40:56 -0500 Subject: [PATCH 402/854] fix(pipeline-template): Do not run on every push --- nf_core/pipeline-template/.github/workflows/linting.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index b2cf30270f..8f9a4173bd 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -4,6 +4,8 @@ name: nf-core linting # that the code meets the nf-core guidelines. {%- raw %} on: push: + branches: + - dev pull_request: release: types: [published] From 908197ad7f4759a8e48a0bda8eaadd362263618b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 28 Oct 2022 09:44:11 -0500 Subject: [PATCH 403/854] ci: Change logic of "on" for testing and linting This has been floating around. What it does is: - Run when code is merged into dev - Run *once* when a PR is created - Run on a release(since it doesn't happen often we might as well check) What this prevents: - The same CI jobs running *twice* on a PR - CI running whenever someone pushes code they're working on that isn't a PR --- .github/workflows/create-lint-wf.yml | 8 +++++++- .github/workflows/create-test-wf.yml | 8 +++++++- .github/workflows/lint-code.yml | 8 +++++++- .github/workflows/pytest.yml | 4 ++++ .github/workflows/tools-api-docs-dev.yml | 8 +++++++- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index c8b4461e0a..97adc7ac71 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -1,5 +1,11 @@ name: Create a pipeline and run nf-core linting -on: [push, pull_request] +on: + push: + branches: + - dev + pull_request: + release: + types: [published] # Cancel if a newer run is started concurrency: diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index c2d1eb70bd..603403b505 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -1,5 +1,11 @@ name: Create a pipeline and test it -on: [push, pull_request] +on: + push: + branches: + - dev + pull_request: + release: + types: [published] # Cancel if a newer run is started concurrency: diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index ee71772b55..a7cda448c4 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -1,5 +1,11 @@ name: Lint tools code formatting -on: [push, pull_request] +on: + push: + branches: + - dev + pull_request: + release: + types: [published] # Cancel if a newer run is started concurrency: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 8e578c8bec..6e8848ed9b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -3,7 +3,11 @@ name: Python tests # Only run if we changed a Python file on: push: + branches: + - dev pull_request: + release: + types: [published] # Cancel if a newer run is started concurrency: diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 2b1099daa3..20508c20a5 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -1,6 +1,12 @@ name: nf-core/tools dev API docs # Run on push and PR to test that docs build -on: [pull_request, push] +on: + push: + branches: + - dev + pull_request: + release: + types: [published] # Cancel if a newer run is started concurrency: From 23c457ce80465911bb28ad44568574ba9c6cf726 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 28 Oct 2022 16:48:26 +0200 Subject: [PATCH 404/854] move and rename prompt_module_version_sha --- nf_core/components/components_install.py | 4 ++- nf_core/components/components_utils.py | 46 ++++++++++++++++++++++++ nf_core/modules/modules_utils.py | 45 ----------------------- nf_core/modules/update.py | 5 +-- 4 files changed, 52 insertions(+), 48 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 99833b54e3..185c75e0d3 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -10,6 +10,7 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME log = logging.getLogger(__name__) @@ -97,8 +98,9 @@ def get_version(component, component_type, sha, prompt, current_version, modules version = sha elif prompt: try: - version = nf_core.modules.modules_utils.prompt_module_version_sha( + version = nf_core.modules.modules_utils.prompt_component_version_sha( component, + component_type, installed_sha=current_version, modules_repo=modules_repo, ) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index e826cc37c8..bbe629adfc 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -67,3 +67,49 @@ def get_repo_type(dir, repo_type=None, use_prompt=True): # It was set on the command line, return what we were given return [dir, repo_type] + + +def prompt_component_version_sha(component_name, component_type, modules_repo, installed_sha=None): + """ + Creates an interactive questionary prompt for selecting the module/subworkflow version + Args: + component_name (str): Module/subworkflow name, + component_type (str): "modules" or "subworkflows", + modules_repo (ModulesRepo): Modules repo the module/subworkflow originate in + installed_sha (str): Optional extra argument to highlight the current installed version + + Returns: + git_sha (str): The selected version of the module/subworkflow + """ + older_commits_choice = questionary.Choice( + title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value="" + ) + git_sha = "" + page_nbr = 1 + + all_commits = modules_repo.get_component_git_log(component_name, subworkflow) + next_page_commits = [next(all_commits, None) for _ in range(10)] + next_page_commits = [commit for commit in next_page_commits if commit is not None] + + while git_sha == "": + commits = next_page_commits + next_page_commits = [next(all_commits, None) for _ in range(10)] + next_page_commits = [commit for commit in next_page_commits if commit is not None] + if all(commit is None for commit in next_page_commits): + next_page_commits = None + + choices = [] + for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): + display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" + message = f"{title} {sha}" + if installed_sha == sha: + message += " (installed version)" + commit_display = [(display_color, message), ("class:choice-default", "")] + choices.append(questionary.Choice(title=commit_display, value=sha)) + if next_page_commits is not None: + choices += [older_commits_choice] + git_sha = questionary.select( + f"Select '{component_name}' commit:", choices=choices, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + page_nbr += 1 + return git_sha diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index 8231320e27..ab633c93df 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -182,48 +182,3 @@ def get_repo_type(dir, repo_type=None, use_prompt=True): # It was set on the command line, return what we were given return [dir, repo_type] - - -def prompt_module_version_sha(module, modules_repo, installed_sha=None): - """ - Creates an interactive questionary prompt for selecting the module version - Args: - module (str): Module name - modules_repo (ModulesRepo): Modules repo the module originate in - installed_sha (str): Optional extra argument to highlight the current installed version - - Returns: - git_sha (str): The selected version of the module - """ - older_commits_choice = questionary.Choice( - title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value="" - ) - git_sha = "" - page_nbr = 1 - - all_commits = modules_repo.get_component_git_log(module, "modules") - next_page_commits = [next(all_commits, None) for _ in range(10)] - next_page_commits = [commit for commit in next_page_commits if commit is not None] - - while git_sha == "": - commits = next_page_commits - next_page_commits = [next(all_commits, None) for _ in range(10)] - next_page_commits = [commit for commit in next_page_commits if commit is not None] - if all(commit is None for commit in next_page_commits): - next_page_commits = None - - choices = [] - for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): - display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" - message = f"{title} {sha}" - if installed_sha == sha: - message += " (installed version)" - commit_display = [(display_color, message), ("class:choice-default", "")] - choices.append(questionary.Choice(title=commit_display, value=sha)) - if next_page_commits is not None: - choices += [older_commits_choice] - git_sha = questionary.select( - f"Select '{module}' commit:", choices=choices, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - page_nbr += 1 - return git_sha diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index fad57ef0e2..592d35b8bb 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -8,6 +8,7 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_utils import prompt_component_version_sha from nf_core.utils import plural_es, plural_s, plural_y from .modules_command import ModuleCommand @@ -151,8 +152,8 @@ def update(self, module=None): if sha is not None: version = sha elif self.prompt: - version = nf_core.modules.modules_utils.prompt_module_version_sha( - module, modules_repo=modules_repo, installed_sha=current_version + version = nf_core.modules.modules_utils.prompt_component_version_sha( + module, "modules", modules_repo=modules_repo, installed_sha=current_version ) else: version = modules_repo.get_latest_component_version(module, "modules") From efee0354de2a644f4d798e58b280b27e41b209cd Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Fri, 28 Oct 2022 10:56:20 -0500 Subject: [PATCH 405/854] Sync gitpod and codespaces. --- .devcontainer/Dockerfile | 34 -------------------------- .devcontainer/add-notice.sh | 20 --------------- .devcontainer/devcontainer.json | 43 ++++++++++----------------------- .devcontainer/noop.txt | 3 --- 4 files changed, 13 insertions(+), 87 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 .devcontainer/add-notice.sh delete mode 100644 .devcontainer/noop.txt diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index c866df8c51..0000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda/.devcontainer/base.Dockerfile - -FROM mcr.microsoft.com/vscode/devcontainers/anaconda:0-3 - -# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 -ARG NODE_VERSION="none" -RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi - -# Copy environment.yml (if found) to a temp location so we update the environment. Also -# copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists. -COPY environment.yml* .devcontainer/noop.txt /tmp/conda-tmp/ -RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; fi \ - && rm -rf /tmp/conda-tmp - -# [Optional] Uncomment this section to install additional OS packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends - -RUN pip install nf-core -RUN conda update -n base -c defaults conda && \ - conda config --add channels defaults && \ - conda config --add channels bioconda && \ - conda config --add channels conda-forge && \ - conda install \ - openjdk=11.0.15 \ - nextflow=22.04.0 \ - pytest-workflow=1.6.0 \ - mamba=0.24.0 \ - pip=22.1.2 \ - black=22.6.0 \ - prettier=2.7.1 \ - -n base && \ - conda clean --all -f -y - diff --git a/.devcontainer/add-notice.sh b/.devcontainer/add-notice.sh deleted file mode 100644 index 0417abb6cd..0000000000 --- a/.devcontainer/add-notice.sh +++ /dev/null @@ -1,20 +0,0 @@ -# Display a notice when not running in GitHub Codespaces - -cat << 'EOF' > /usr/local/etc/vscode-dev-containers/conda-notice.txt -When using "conda" from outside of GitHub Codespaces, note the Anaconda repository -contains restrictions on commercial use that may impact certain organizations. See -https://aka.ms/vscode-remote/conda/anaconda - -EOF - -notice_script="$(cat << 'EOF' -if [ -t 1 ] && [ "${IGNORE_NOTICE}" != "true" ] && [ "${TERM_PROGRAM}" = "vscode" ] && [ "${CODESPACES}" != "true" ] && [ ! -f "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed" ]; then - cat "/usr/local/etc/vscode-dev-containers/conda-notice.txt" - mkdir -p "$HOME/.config/vscode-dev-containers" - ((sleep 10s; touch "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed") &) -fi -EOF -)" - -echo "${notice_script}" | tee -a /etc/bash.bashrc >> /etc/zsh/zshrc - diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f8df1d2be2..dfc011a4cd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,14 +1,8 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda { - "name": "Anaconda (Python 3)", - "build": { - "context": "..", - "dockerfile": "Dockerfile", - "args": { - "NODE_VERSION": "lts/*" - } - }, + "name": "nfcore", + "image": "nfcore/gitpod:latest", + "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e .", + "remoteUser": "gitpod", // Configure tool-specific properties. "customizations": { @@ -31,26 +25,15 @@ "extensions": [ "ms-python.python", "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack" + "nf-core.nf-core-extensionpack", + "codezombiech.gitignore", + "esbenp.prettier-vscode", + "EditorConfig.EditorConfig", + "Gruntfuggly.todo-tree", + "mechatroner.rainbow-csv", + "derwat.indent-rainbow", + "streetsidesoftware.code-spell-checker" ] } - }, - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "python --version", - - // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", - "features": { - "docker-in-docker": "20.10", - "git": "os-provided", - "ghcr.io/devcontainers/features/conda:1": {}, - "ghcr.io/devcontainers/features/docker-from-docker:1": {}, - "ghcr.io/devcontainers/features/git:1": {} } -} - - +} \ No newline at end of file diff --git a/.devcontainer/noop.txt b/.devcontainer/noop.txt deleted file mode 100644 index 09c5ae9b86..0000000000 --- a/.devcontainer/noop.txt +++ /dev/null @@ -1,3 +0,0 @@ -This file copied into the container along with environment.yml* from the parent -folder. This file is included to prevents the Dockerfile COPY instruction from -failing if no environment.yml is found. From 3274319574c117004256891866c6b43add22892a Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Fri, 28 Oct 2022 11:29:10 -0500 Subject: [PATCH 406/854] Fix build of nfcore tools. --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index dfc011a4cd..678eea8d20 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "name": "nfcore", "image": "nfcore/gitpod:latest", - "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e .", + "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e ../", "remoteUser": "gitpod", // Configure tool-specific properties. From 1f6a256ad934cac6612f88379cac7ea136ecd5ae Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Fri, 28 Oct 2022 11:32:26 -0500 Subject: [PATCH 407/854] Sync gitpod and codespaces. --- .../.devcontainer/Dockerfile | 34 --------------- .../.devcontainer/add-notice.sh | 20 --------- .../.devcontainer/devcontainer.json | 42 ++++++------------- .../pipeline-template/.devcontainer/noop.txt | 3 -- 4 files changed, 12 insertions(+), 87 deletions(-) delete mode 100644 nf_core/pipeline-template/.devcontainer/Dockerfile delete mode 100644 nf_core/pipeline-template/.devcontainer/add-notice.sh delete mode 100644 nf_core/pipeline-template/.devcontainer/noop.txt diff --git a/nf_core/pipeline-template/.devcontainer/Dockerfile b/nf_core/pipeline-template/.devcontainer/Dockerfile deleted file mode 100644 index c866df8c51..0000000000 --- a/nf_core/pipeline-template/.devcontainer/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda/.devcontainer/base.Dockerfile - -FROM mcr.microsoft.com/vscode/devcontainers/anaconda:0-3 - -# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 -ARG NODE_VERSION="none" -RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi - -# Copy environment.yml (if found) to a temp location so we update the environment. Also -# copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists. -COPY environment.yml* .devcontainer/noop.txt /tmp/conda-tmp/ -RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; fi \ - && rm -rf /tmp/conda-tmp - -# [Optional] Uncomment this section to install additional OS packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends - -RUN pip install nf-core -RUN conda update -n base -c defaults conda && \ - conda config --add channels defaults && \ - conda config --add channels bioconda && \ - conda config --add channels conda-forge && \ - conda install \ - openjdk=11.0.15 \ - nextflow=22.04.0 \ - pytest-workflow=1.6.0 \ - mamba=0.24.0 \ - pip=22.1.2 \ - black=22.6.0 \ - prettier=2.7.1 \ - -n base && \ - conda clean --all -f -y - diff --git a/nf_core/pipeline-template/.devcontainer/add-notice.sh b/nf_core/pipeline-template/.devcontainer/add-notice.sh deleted file mode 100644 index 0417abb6cd..0000000000 --- a/nf_core/pipeline-template/.devcontainer/add-notice.sh +++ /dev/null @@ -1,20 +0,0 @@ -# Display a notice when not running in GitHub Codespaces - -cat << 'EOF' > /usr/local/etc/vscode-dev-containers/conda-notice.txt -When using "conda" from outside of GitHub Codespaces, note the Anaconda repository -contains restrictions on commercial use that may impact certain organizations. See -https://aka.ms/vscode-remote/conda/anaconda - -EOF - -notice_script="$(cat << 'EOF' -if [ -t 1 ] && [ "${IGNORE_NOTICE}" != "true" ] && [ "${TERM_PROGRAM}" = "vscode" ] && [ "${CODESPACES}" != "true" ] && [ ! -f "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed" ]; then - cat "/usr/local/etc/vscode-dev-containers/conda-notice.txt" - mkdir -p "$HOME/.config/vscode-dev-containers" - ((sleep 10s; touch "$HOME/.config/vscode-dev-containers/conda-notice-already-displayed") &) -fi -EOF -)" - -echo "${notice_script}" | tee -a /etc/bash.bashrc >> /etc/zsh/zshrc - diff --git a/nf_core/pipeline-template/.devcontainer/devcontainer.json b/nf_core/pipeline-template/.devcontainer/devcontainer.json index f8df1d2be2..0673bad888 100644 --- a/nf_core/pipeline-template/.devcontainer/devcontainer.json +++ b/nf_core/pipeline-template/.devcontainer/devcontainer.json @@ -1,14 +1,7 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3-anaconda { - "name": "Anaconda (Python 3)", - "build": { - "context": "..", - "dockerfile": "Dockerfile", - "args": { - "NODE_VERSION": "lts/*" - } - }, + "name": "nfcore", + "image": "nfcore/gitpod:latest", + "remoteUser": "gitpod", // Configure tool-specific properties. "customizations": { @@ -31,26 +24,15 @@ "extensions": [ "ms-python.python", "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack" + "nf-core.nf-core-extensionpack", + "codezombiech.gitignore", + "esbenp.prettier-vscode", + "EditorConfig.EditorConfig", + "Gruntfuggly.todo-tree", + "mechatroner.rainbow-csv", + "derwat.indent-rainbow", + "streetsidesoftware.code-spell-checker" ] } - }, - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "python --version", - - // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", - "features": { - "docker-in-docker": "20.10", - "git": "os-provided", - "ghcr.io/devcontainers/features/conda:1": {}, - "ghcr.io/devcontainers/features/docker-from-docker:1": {}, - "ghcr.io/devcontainers/features/git:1": {} } -} - - +} \ No newline at end of file diff --git a/nf_core/pipeline-template/.devcontainer/noop.txt b/nf_core/pipeline-template/.devcontainer/noop.txt deleted file mode 100644 index 09c5ae9b86..0000000000 --- a/nf_core/pipeline-template/.devcontainer/noop.txt +++ /dev/null @@ -1,3 +0,0 @@ -This file copied into the container along with environment.yml* from the parent -folder. This file is included to prevents the Dockerfile COPY instruction from -failing if no environment.yml is found. From a51bfec5943016ab5037e77b3c2f69e594e59732 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 28 Oct 2022 17:39:34 +0000 Subject: [PATCH 408/854] [automated] Fix code linting --- .devcontainer/devcontainer.json | 68 +++++++++---------- .github/CONTRIBUTING.md | 7 +- .../.devcontainer/devcontainer.json | 66 +++++++++--------- 3 files changed, 72 insertions(+), 69 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 678eea8d20..31f3628849 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,39 +1,39 @@ { "name": "nfcore", "image": "nfcore/gitpod:latest", - "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e ../", + "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e ../", "remoteUser": "gitpod", - // Configure tool-specific properties. - "customizations": { - // Configure properties specific to VS Code. - "vscode": { - // Set *default* container specific settings.json values on container create. - "settings": { - "python.defaultInterpreterPath": "/opt/conda/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", - "python.formatting.yapfPath": "/opt/conda/bin/yapf", - "python.linting.flake8Path": "/opt/conda/bin/flake8", - "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", - "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack", - "codezombiech.gitignore", - "esbenp.prettier-vscode", - "EditorConfig.EditorConfig", - "Gruntfuggly.todo-tree", - "mechatroner.rainbow-csv", - "derwat.indent-rainbow", - "streetsidesoftware.code-spell-checker" - ] - } - } -} \ No newline at end of file + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/opt/conda/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", + "python.formatting.yapfPath": "/opt/conda/bin/yapf", + "python.linting.flake8Path": "/opt/conda/bin/flake8", + "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", + "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", + "python.linting.pylintPath": "/opt/conda/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "nf-core.nf-core-extensionpack", + "codezombiech.gitignore", + "esbenp.prettier-vscode", + "EditorConfig.EditorConfig", + "Gruntfuggly.todo-tree", + "mechatroner.rainbow-csv", + "derwat.indent-rainbow", + "streetsidesoftware.code-spell-checker" + ] + } + } +} diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 851d88cbcc..b3c22ef484 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -129,14 +129,17 @@ nf-core lint nf-core-testpipeline ``` ## Codespaces + This repo includes a devcontainer configuration which will create a GitHub Codespace for Nextflow development! Devcontainer specs: + - [DevContainer config](.devcontainer/devcontainer.json) - [Dockerfile](.devcontainer/Dockerfile) # To get started: + - Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) - Tools installed - - nf-core - - nextflow + - nf-core + - nextflow diff --git a/nf_core/pipeline-template/.devcontainer/devcontainer.json b/nf_core/pipeline-template/.devcontainer/devcontainer.json index 0673bad888..d4c8fa8fa5 100644 --- a/nf_core/pipeline-template/.devcontainer/devcontainer.json +++ b/nf_core/pipeline-template/.devcontainer/devcontainer.json @@ -3,36 +3,36 @@ "image": "nfcore/gitpod:latest", "remoteUser": "gitpod", - // Configure tool-specific properties. - "customizations": { - // Configure properties specific to VS Code. - "vscode": { - // Set *default* container specific settings.json values on container create. - "settings": { - "python.defaultInterpreterPath": "/opt/conda/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", - "python.formatting.yapfPath": "/opt/conda/bin/yapf", - "python.linting.flake8Path": "/opt/conda/bin/flake8", - "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", - "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack", - "codezombiech.gitignore", - "esbenp.prettier-vscode", - "EditorConfig.EditorConfig", - "Gruntfuggly.todo-tree", - "mechatroner.rainbow-csv", - "derwat.indent-rainbow", - "streetsidesoftware.code-spell-checker" - ] - } - } -} \ No newline at end of file + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/opt/conda/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", + "python.formatting.yapfPath": "/opt/conda/bin/yapf", + "python.linting.flake8Path": "/opt/conda/bin/flake8", + "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", + "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", + "python.linting.pylintPath": "/opt/conda/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "nf-core.nf-core-extensionpack", + "codezombiech.gitignore", + "esbenp.prettier-vscode", + "EditorConfig.EditorConfig", + "Gruntfuggly.todo-tree", + "mechatroner.rainbow-csv", + "derwat.indent-rainbow", + "streetsidesoftware.code-spell-checker" + ] + } + } +} From 2fc9c65e72e2033389147254dc977d81f5d6416a Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 28 Oct 2022 18:10:12 +0000 Subject: [PATCH 409/854] update nextflow version --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index b8991ae823..2453ae8405 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -26,7 +26,7 @@ RUN conda update -n base -c defaults conda && \ conda config --add channels conda-forge && \ conda install \ openjdk=11.0.15 \ - nextflow=22.04.0 \ + nextflow=22.10.0 \ pytest-workflow=1.6.0 \ mamba=0.24.0 \ pip=22.1.2 \ From 4ced477fa6e689d56ac2a67abccf8e7e1e60ee32 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 28 Oct 2022 18:10:38 +0000 Subject: [PATCH 410/854] update mamba version --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 2453ae8405..a977d7974f 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -28,7 +28,7 @@ RUN conda update -n base -c defaults conda && \ openjdk=11.0.15 \ nextflow=22.10.0 \ pytest-workflow=1.6.0 \ - mamba=0.24.0 \ + mamba=0.27.0 \ pip=22.1.2 \ black=22.6.0 \ prettier=2.7.1 \ From 76d82db3f84d52c6799970f849211fb622684aa7 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 28 Oct 2022 18:11:30 +0000 Subject: [PATCH 411/854] update pip version --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index a977d7974f..5f5e928285 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -29,7 +29,7 @@ RUN conda update -n base -c defaults conda && \ nextflow=22.10.0 \ pytest-workflow=1.6.0 \ mamba=0.27.0 \ - pip=22.1.2 \ + pip=22.3 \ black=22.6.0 \ prettier=2.7.1 \ -n base && \ From d23a140d453165a30cde98b07ac451c73735d3e4 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 28 Oct 2022 18:12:10 +0000 Subject: [PATCH 412/854] update black version --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 5f5e928285..0142a0477e 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -30,7 +30,7 @@ RUN conda update -n base -c defaults conda && \ pytest-workflow=1.6.0 \ mamba=0.27.0 \ pip=22.3 \ - black=22.6.0 \ + black=22.10.0 \ prettier=2.7.1 \ -n base && \ conda clean --all -f -y From 29de649f76bdc750bf1835ce52d10f0e6c15feab Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 28 Oct 2022 18:13:48 +0000 Subject: [PATCH 413/854] add nf-test --- nf_core/gitpod/gitpod.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 0142a0477e..e67e64f5a7 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -27,6 +27,7 @@ RUN conda update -n base -c defaults conda && \ conda install \ openjdk=11.0.15 \ nextflow=22.10.0 \ + nf-test=0.7.0 \ pytest-workflow=1.6.0 \ mamba=0.27.0 \ pip=22.3 \ From bdb223b3ad667d0c3fd3fc0ca45f36caf27d4f6b Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 28 Oct 2022 18:14:59 +0000 Subject: [PATCH 414/854] update openjdk version --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index e67e64f5a7..91eaa88c6c 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -25,7 +25,7 @@ RUN conda update -n base -c defaults conda && \ conda config --add channels bioconda && \ conda config --add channels conda-forge && \ conda install \ - openjdk=11.0.15 \ + openjdk=17.0.3 \ nextflow=22.10.0 \ nf-test=0.7.0 \ pytest-workflow=1.6.0 \ From fbb1759a036444f13b9e96e77812a82d5a50e0ab Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 29 Oct 2022 12:25:51 +0200 Subject: [PATCH 415/854] use f-string instead of format --- nf_core/lint_utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 19d253c53c..b412772259 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -37,16 +37,18 @@ def print_joint_summary(lint_obj, module_lint_obj): def print_fixes(lint_obj, module_lint_obj): """Prints available and applied fixes""" - if len(lint_obj.could_fix): - fix_cmd = "nf-core lint {} --fix {}".format( - "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}", " --fix ".join(lint_obj.could_fix) - ) + fixe_flags = " ".join([f"--fix {file}" for file in lint_obj.could_fix]) + + if fixe_flags: + lint_dir = "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}" console.print( - f"\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n[blue] {fix_cmd}\n" + "\nTip: Some of these linting errors can automatically be resolved with the following command:" + f"\n\n[blue] nf-core lint {lint_dir} {fixe_flags}\n" ) if len(lint_obj.fix): console.print( - "Automatic fixes applied. Please check with 'git diff' and revert any changes you do not want with 'git checkout '." + "Automatic fixes applied. Please check with 'git diff' and revert " + "any changes you do not want with 'git checkout '." ) From b1882d8a4a13abfaebb4fdfcef57b7edd1f3151f Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 29 Oct 2022 12:28:16 +0200 Subject: [PATCH 416/854] use pre-commit installed Prettier if not already installed --- nf_core/lint_utils.py | 50 +++++++++++++++++++++++++++++++++++-------- requirements-dev.txt | 1 - requirements.txt | 1 + 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index b412772259..61b472fcf6 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,4 +1,5 @@ import logging +import shutil import subprocess import rich @@ -53,7 +54,7 @@ def print_fixes(lint_obj, module_lint_obj): def run_prettier_on_file(file): - """Runs Prettier on a file if Prettier is installed. + """Run Prettier on a file. Args: file (Path | str): A file identifier as a string or pathlib.Path. @@ -62,12 +63,43 @@ def run_prettier_on_file(file): If Prettier is not installed, a warning is logged. """ - try: - subprocess.run( - ["prettier", "--write", file], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, + _prettier_installed = not shutil.which("prettier") is None + _pre_commit_installed = not shutil.which("pre-commit") is None + + if _prettier_installed: + _run_prettier_on_file(file) + elif _pre_commit_installed: + _run_pre_commit_prettier_on_file(file) + else: + log.warning( + "Neither Prettier nor the prettier pre-commit hook are available. At least one of them is required." ) - except FileNotFoundError: - log.warning("Prettier is not installed. Please install it and run it on the pipeline to fix linting issues.") + + +def _run_prettier_on_file(file): + """Run natively installed Prettier on a file.""" + + subprocess.run( + ["prettier", "--write", file], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +def _run_pre_commit_prettier_on_file(file): + """Runs pre-commit hook prettier on a file if pre-commit is installed. + + Args: + file (Path | str): A file identifier as a string or pathlib.Path. + + Warns: + If Prettier is not installed, a warning is logged. + """ + + subprocess.run( + ["pre-commit", "run", "prettier", file], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8fc8d1c7de..3809ae1c1d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,6 @@ black isort myst_parser -pre-commit pytest-cov pytest-datafiles Sphinx diff --git a/requirements.txt b/requirements.txt index 0a4a5fb7e7..fcc5130f8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ jinja2 jsonschema>=3.0 markdown>=3.3 packaging +pre-commit prompt_toolkit>=3.0.3 pytest>=7.0.0 pytest-workflow>=1.6.0 From a9a9be9031ecd6149dd2976aeebc90077505c3fd Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 29 Oct 2022 12:57:22 +0200 Subject: [PATCH 417/854] ship and use the nf-core pre-commit config for linting --- MANIFEST.in | 1 + nf_core/lint_utils.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 384a7b5441..7363b92974 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ graft nf_core/module-template graft nf_core/pipeline-template graft nf_core/subworkflow-template include requirements.txt +include .pre-commit-config.yaml diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 61b472fcf6..d50bb51e41 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,6 +1,7 @@ import logging import shutil import subprocess +from pathlib import Path import rich from rich.console import Console @@ -97,8 +98,10 @@ def _run_pre_commit_prettier_on_file(file): If Prettier is not installed, a warning is logged. """ + nf_core_pre_commit_config = Path(nf_core.__file__).parent.parent / ".pre-commit-config.yaml" + subprocess.run( - ["pre-commit", "run", "prettier", file], + ["pre-commit", "run", f"--config {nf_core_pre_commit_config}", "prettier", f"--files {file}"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, From 5860858ea48591a93b3318d4f105ab2f31c66f3e Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 29 Oct 2022 14:29:04 +0200 Subject: [PATCH 418/854] fix typo in variable name --- nf_core/lint_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index d50bb51e41..bf9e9519ae 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -39,9 +39,9 @@ def print_joint_summary(lint_obj, module_lint_obj): def print_fixes(lint_obj, module_lint_obj): """Prints available and applied fixes""" - fixe_flags = " ".join([f"--fix {file}" for file in lint_obj.could_fix]) + fix_flags = " ".join([f"--fix {file}" for file in lint_obj.could_fix]) - if fixe_flags: + if fix_flags: lint_dir = "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}" console.print( "\nTip: Some of these linting errors can automatically be resolved with the following command:" From 5e9fcae5cae8c04e866c4a287dd4e33da2b8c654 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 27 Oct 2022 00:24:51 +0200 Subject: [PATCH 419/854] remove useless shebang lines --- nf_core/__init__.py | 1 - nf_core/bump_version.py | 1 - nf_core/create.py | 1 - nf_core/download.py | 1 - nf_core/launch.py | 1 - nf_core/licences.py | 1 - nf_core/lint/__init__.py | 1 - nf_core/lint/actions_awsfulltest.py | 2 -- nf_core/lint/actions_awstest.py | 2 -- nf_core/lint/actions_ci.py | 2 -- nf_core/lint/actions_schema_validation.py | 2 -- nf_core/lint/files_exist.py | 2 -- nf_core/lint/files_unchanged.py | 2 -- nf_core/lint/merge_markers.py | 2 -- nf_core/lint/modules_json.py | 2 -- nf_core/lint/modules_structure.py | 2 -- nf_core/lint/multiqc_config.py | 2 -- nf_core/lint/nextflow_config.py | 2 -- nf_core/lint/pipeline_name_conventions.py | 3 --- nf_core/lint/pipeline_todos.py | 2 -- nf_core/lint/readme.py | 2 -- nf_core/lint/schema_description.py | 2 -- nf_core/lint/schema_lint.py | 2 -- nf_core/lint/schema_params.py | 2 -- nf_core/lint/template_strings.py | 2 -- nf_core/lint/version_consistency.py | 2 -- nf_core/list.py | 1 - nf_core/modules/create.py | 1 - nf_core/modules/lint/__init__.py | 1 - nf_core/modules/lint/main_nf.py | 1 - nf_core/modules/lint/meta_yml.py | 2 -- nf_core/modules/lint/module_deprecations.py | 1 - nf_core/modules/lint/module_todos.py | 1 - nf_core/modules/lint/module_version.py | 1 - nf_core/modules/modules_test.py | 1 - nf_core/modules/test_yml_builder.py | 1 - nf_core/refgenie.py | 1 - nf_core/schema.py | 1 - nf_core/subworkflows/create.py | 1 - nf_core/subworkflows/test_yml_builder.py | 1 - nf_core/sync.py | 1 - nf_core/utils.py | 1 - tests/lint/actions_awsfulltest.py | 2 -- tests/lint/actions_awstest.py | 2 -- tests/lint/actions_ci.py | 2 -- tests/lint/actions_schema_validation.py | 2 -- tests/lint/files_exist.py | 2 -- tests/lint/merge_markers.py | 2 -- tests/test_bump_version.py | 1 - tests/test_cli.py | 1 - tests/test_create.py | 1 - tests/test_download.py | 1 - tests/test_launch.py | 1 - tests/test_licenses.py | 1 - tests/test_lint.py | 1 - tests/test_list.py | 1 - tests/test_modules.py | 1 - tests/test_refgenie.py | 1 - tests/test_schema.py | 1 - tests/test_subworkflows.py | 1 - tests/test_sync.py | 1 - tests/test_utils.py | 1 - tests/utils.py | 2 -- 63 files changed, 91 deletions(-) diff --git a/nf_core/__init__.py b/nf_core/__init__.py index 42c3f188f3..e333335280 100644 --- a/nf_core/__init__.py +++ b/nf_core/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Main nf_core module file. Shouldn't do much, as everything is under subcommands. diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 53766678b0..1f38775bb6 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Bumps the version number in all appropriate files for a nf-core pipeline. """ diff --git a/nf_core/create.py b/nf_core/create.py index bd00dd1517..42bdf4187c 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Creates a nf-core pipeline matching the current organization's specification based on a template. """ diff --git a/nf_core/download.py b/nf_core/download.py index 923d953fae..5e7f4edb2f 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Downloads a nf-core pipeline to the local file system.""" from __future__ import print_function diff --git a/nf_core/launch.py b/nf_core/launch.py index d57d9f112f..7b2648458e 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Launch a pipeline, interactively collecting params """ from __future__ import print_function diff --git a/nf_core/licences.py b/nf_core/licences.py index 2e65462a1d..3b188861eb 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Lists software licences for a given workflow.""" from __future__ import print_function diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 5ea01142e5..a0de378d82 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Linting policy for nf-core pipeline projects. Tests Nextflow-based pipelines to check that they adhere to diff --git a/nf_core/lint/actions_awsfulltest.py b/nf_core/lint/actions_awsfulltest.py index e021ebd384..e8e1c951b1 100644 --- a/nf_core/lint/actions_awsfulltest.py +++ b/nf_core/lint/actions_awsfulltest.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import yaml diff --git a/nf_core/lint/actions_awstest.py b/nf_core/lint/actions_awstest.py index 4f27cbd765..ccdf0abf6a 100644 --- a/nf_core/lint/actions_awstest.py +++ b/nf_core/lint/actions_awstest.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import yaml diff --git a/nf_core/lint/actions_ci.py b/nf_core/lint/actions_ci.py index 4d6b0e6dfe..b4c10f3a74 100644 --- a/nf_core/lint/actions_ci.py +++ b/nf_core/lint/actions_ci.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import re diff --git a/nf_core/lint/actions_schema_validation.py b/nf_core/lint/actions_schema_validation.py index 7ded008cfc..9d49b84c6b 100644 --- a/nf_core/lint/actions_schema_validation.py +++ b/nf_core/lint/actions_schema_validation.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import glob import logging import os diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 2a1dee62c9..eb8c04916a 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import logging import os diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 8b63a5769e..cadced5483 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import filecmp import logging import os diff --git a/nf_core/lint/merge_markers.py b/nf_core/lint/merge_markers.py index 75fbf931bf..f33a5095d8 100644 --- a/nf_core/lint/merge_markers.py +++ b/nf_core/lint/merge_markers.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import fnmatch import io import logging diff --git a/nf_core/lint/modules_json.py b/nf_core/lint/modules_json.py index 3e97eed043..dd0a59d558 100644 --- a/nf_core/lint/modules_json.py +++ b/nf_core/lint/modules_json.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from pathlib import Path from nf_core.modules.modules_json import ModulesJson diff --git a/nf_core/lint/modules_structure.py b/nf_core/lint/modules_structure.py index 830e0d1f16..f0e13e0346 100644 --- a/nf_core/lint/modules_structure.py +++ b/nf_core/lint/modules_structure.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import logging import os from pathlib import Path diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 37580a1f11..3378efce5f 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import yaml diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 635e33cfb5..cf0fbf4d40 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import logging import os import re diff --git a/nf_core/lint/pipeline_name_conventions.py b/nf_core/lint/pipeline_name_conventions.py index e1ecad0be2..7fb6ffca0f 100644 --- a/nf_core/lint/pipeline_name_conventions.py +++ b/nf_core/lint/pipeline_name_conventions.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python - - def pipeline_name_conventions(self): """Checks that the pipeline name adheres to nf-core conventions. diff --git a/nf_core/lint/pipeline_todos.py b/nf_core/lint/pipeline_todos.py index 91a7cf6307..890e227fa1 100644 --- a/nf_core/lint/pipeline_todos.py +++ b/nf_core/lint/pipeline_todos.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import fnmatch import io import logging diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index 99def0a204..a85a0979d3 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import re diff --git a/nf_core/lint/schema_description.py b/nf_core/lint/schema_description.py index 3a670e5f70..ca22f266ab 100644 --- a/nf_core/lint/schema_description.py +++ b/nf_core/lint/schema_description.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import nf_core.schema diff --git a/nf_core/lint/schema_lint.py b/nf_core/lint/schema_lint.py index f7c1b11048..178063d5dd 100644 --- a/nf_core/lint/schema_lint.py +++ b/nf_core/lint/schema_lint.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import logging import nf_core.schema diff --git a/nf_core/lint/schema_params.py b/nf_core/lint/schema_params.py index 6b32535738..9280fe4703 100644 --- a/nf_core/lint/schema_params.py +++ b/nf_core/lint/schema_params.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import nf_core.schema diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index 17886cec3f..436abe7b2b 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import io import mimetypes import re diff --git a/nf_core/lint/version_consistency.py b/nf_core/lint/version_consistency.py index 89a8751af6..fa5b50de01 100644 --- a/nf_core/lint/version_consistency.py +++ b/nf_core/lint/version_consistency.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os diff --git a/nf_core/list.py b/nf_core/list.py index 62c925ee47..b4353050e6 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Lists available nf-core pipelines and versions.""" from __future__ import print_function diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 1b1eeba5ea..b96a2422b2 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ The ModuleCreate class handles generating of module templates """ diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index f4ab5583ca..d8e2561cef 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Code for linting modules in the nf-core/modules repository and in nf-core pipelines diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index e396e7e813..f0098c8ab7 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Lint the main.nf file of a module """ diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 77df16deb2..d6ec296999 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - from pathlib import Path import yaml diff --git a/nf_core/modules/lint/module_deprecations.py b/nf_core/modules/lint/module_deprecations.py index f7e8761c75..aa4cabd7b2 100644 --- a/nf_core/modules/lint/module_deprecations.py +++ b/nf_core/modules/lint/module_deprecations.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import logging import os diff --git a/nf_core/modules/lint/module_todos.py b/nf_core/modules/lint/module_todos.py index 90af44987e..8b685a7172 100644 --- a/nf_core/modules/lint/module_todos.py +++ b/nf_core/modules/lint/module_todos.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import logging from nf_core.lint.pipeline_todos import pipeline_todos diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index de77f31cf6..d8ec8e1154 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Verify that a module has a correct entry in the modules.json file """ diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index 92227b992b..472d0495b0 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ The ModulesTest class runs the tests locally """ diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index ce5bb9e7b0..1b0b5e89ac 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ The ModulesTestYmlBuilder class handles automatic generation of the modules test.yml file along with running the tests and creating md5 sums diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py index e8d421d176..a10e4fecdf 100644 --- a/nf_core/refgenie.py +++ b/nf_core/refgenie.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Update a nextflow.config file with refgenie genomes """ diff --git a/nf_core/schema.py b/nf_core/schema.py index 04e18409cb..0be825fea2 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Code to deal with pipeline JSON Schema """ from __future__ import print_function diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 7181498ef5..69c430dcc8 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ The SubworkflowCreate class handles generating of subworkflow templates """ diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 59729ba41d..ac8f80c6b9 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ The ModulesTestYmlBuilder class handles automatic generation of the modules test.yml file along with running the tests and creating md5 sums diff --git a/nf_core/sync.py b/nf_core/sync.py index 5aabd8eb51..2384ad4de7 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Synchronise a pipeline TEMPLATE branch with the template. """ diff --git a/nf_core/utils.py b/nf_core/utils.py index 57907109f9..f5a464b3c2 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Common utility functions for the nf-core python package. """ diff --git a/tests/lint/actions_awsfulltest.py b/tests/lint/actions_awsfulltest.py index 234628227d..30293e31a4 100644 --- a/tests/lint/actions_awsfulltest.py +++ b/tests/lint/actions_awsfulltest.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import yaml diff --git a/tests/lint/actions_awstest.py b/tests/lint/actions_awstest.py index 45d7e66c3d..0e19f781aa 100644 --- a/tests/lint/actions_awstest.py +++ b/tests/lint/actions_awstest.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import yaml diff --git a/tests/lint/actions_ci.py b/tests/lint/actions_ci.py index 81eb26f22c..b6e58889f4 100644 --- a/tests/lint/actions_ci.py +++ b/tests/lint/actions_ci.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import yaml diff --git a/tests/lint/actions_schema_validation.py b/tests/lint/actions_schema_validation.py index d71603a56e..48bb07e4dd 100644 --- a/tests/lint/actions_schema_validation.py +++ b/tests/lint/actions_schema_validation.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import yaml diff --git a/tests/lint/files_exist.py b/tests/lint/files_exist.py index 02686e9add..4e5e4d3c2b 100644 --- a/tests/lint/files_exist.py +++ b/tests/lint/files_exist.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import nf_core.lint diff --git a/tests/lint/merge_markers.py b/tests/lint/merge_markers.py index 6ea2882417..be0d076757 100644 --- a/tests/lint/merge_markers.py +++ b/tests/lint/merge_markers.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import nf_core.lint diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 96eb1240a0..658a2339d4 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Some tests covering the bump_version code. """ import os diff --git a/tests/test_cli.py b/tests/test_cli.py index d8d9eb29c7..e01e048e3e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the command-line code. Most tests check the cli arguments are passed along and that some action is diff --git a/tests/test_create.py b/tests/test_create.py index 5f732fc180..baac509d74 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Some tests covering the pipeline creation sub command. """ import os diff --git a/tests/test_download.py b/tests/test_download.py index feb486e2c8..4577a83992 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Tests for the download subcommand of nf-core tools """ diff --git a/tests/test_launch.py b/tests/test_launch.py index 4d4ed79649..de597127a2 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the pipeline launch code. """ diff --git a/tests/test_licenses.py b/tests/test_licenses.py index a2dde1639b..4fb58a107c 100644 --- a/tests/test_licenses.py +++ b/tests/test_licenses.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Some tests covering the pipeline creation sub command. """ # import json diff --git a/tests/test_lint.py b/tests/test_lint.py index db9bf0fc36..5012d20683 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Some tests covering the linting code. """ import fnmatch diff --git a/tests/test_list.py b/tests/test_list.py index 2acef31a76..70af3fada5 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the workflow listing code. """ diff --git a/tests/test_modules.py b/tests/test_modules.py index e1b9609699..d52386665c 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the modules commands """ diff --git a/tests/test_refgenie.py b/tests/test_refgenie.py index 9314b44eef..1ff2683416 100644 --- a/tests/test_refgenie.py +++ b/tests/test_refgenie.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the refgenie integration code """ diff --git a/tests/test_schema.py b/tests/test_schema.py index 10313c81f3..d3b4fda817 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the pipeline schema code. """ diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index ea4fa986dd..f7b0a567a9 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the subworkflows commands """ diff --git a/tests/test_sync.py b/tests/test_sync.py index 0cedf51698..f0f6c7edca 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the sync command """ diff --git a/tests/test_utils.py b/tests/test_utils.py index b43d6e8ff2..2ab5b64bfc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering for utility functions. """ diff --git a/tests/utils.py b/tests/utils.py index 483f817f20..2342cdc08c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Helper functions for tests """ From fec4a2df5db072b33fb473e49561dde012b7cd29 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 31 Oct 2022 11:48:53 +0100 Subject: [PATCH 420/854] fix name collision if input name is provided, add default value to run command at the end of launch --- nf_core/launch.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index d57d9f112f..2287ae76f1 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -120,13 +120,19 @@ def launch_pipeline(self): # Check if the output file exists already if os.path.exists(self.params_out): - log.warning(f"Parameter output file already exists! {os.path.relpath(self.params_out)}") - if Confirm.ask("[yellow]Do you want to overwrite this file?"): - os.remove(self.params_out) - log.info(f"Deleted {self.params_out}\n") + # if params_in has the same name as params_out, don't ask to overwrite + if self.params_in and os.path.abspath(self.params_in) == os.path.abspath(self.params_out): + log.warning( + f"The parameter input file has the same name as the output file! {os.path.relpath(self.params_out)} will be overwritten." + ) else: - log.info("Exiting. Use --params-out to specify a custom filename.") - return False + log.warning(f"Parameter output file already exists! {os.path.relpath(self.params_out)}") + if Confirm.ask("[yellow]Do you want to overwrite this file?"): + os.remove(self.params_out) + log.info(f"Deleted {self.params_out}\n") + else: + log.info("Exiting. Use --params-out to specify a custom filename.") + return False log.info( "NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles\n" @@ -716,6 +722,6 @@ def launch_workflow(self): """Launch nextflow if required""" log.info(f"[bold underline]Nextflow command:[/]\n[magenta]{self.nextflow_cmd}\n\n") - if Confirm.ask("Do you want to run this command now? "): + if Confirm.ask("Do you want to run this command now? ", default=True): log.info("Launching workflow! :rocket:") subprocess.call(self.nextflow_cmd, shell=True) From eff0721fd5535ee666214e31d36e8dd71f68cf10 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 31 Oct 2022 11:50:57 +0100 Subject: [PATCH 421/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e51660e52c..61ca21ffed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Mock biocontainers and anaconda api calls in modules and subworkflows tests [#1967](https://github.com/nf-core/tools/pull/1967) - Run tests with Python 3.11 ([#1970](https://github.com/nf-core/tools/pull/1970)) - Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971)) +- Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out` ### Modules From e0baf780a077f1b7e3567d7bd9cf0ac3decdb510 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 31 Oct 2022 11:57:19 +0100 Subject: [PATCH 422/854] still ask before overwriting --- nf_core/launch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 2287ae76f1..53fce1f43f 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -127,12 +127,13 @@ def launch_pipeline(self): ) else: log.warning(f"Parameter output file already exists! {os.path.relpath(self.params_out)}") - if Confirm.ask("[yellow]Do you want to overwrite this file?"): + if Confirm.ask("[yellow]Do you want to overwrite this file?"): + if not (self.params_in and os.path.abspath(self.params_in) == os.path.abspath(self.params_out)): os.remove(self.params_out) log.info(f"Deleted {self.params_out}\n") - else: - log.info("Exiting. Use --params-out to specify a custom filename.") - return False + else: + log.info("Exiting. Use --params-out to specify a custom output filename.") + return False log.info( "NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles\n" From 798910224a3fd05497b3f6a5971ce54410f9dcb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 2 Nov 2022 10:45:20 +0100 Subject: [PATCH 423/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/components/components_install.py | 2 +- nf_core/modules/update.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 185c75e0d3..18e30e4e42 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -115,7 +115,7 @@ def get_version(component, component_type, sha, prompt, current_version, modules def clean_modules_json(component, component_type, modules_repo, modules_json): """ - Remove installed version of module/subworkflow + Remove installed version of module/subworkflow from modules.json """ for repo_url, repo_content in modules_json.modules_json["repos"].items(): for dir, dir_components in repo_content[component_type].items(): diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 592d35b8bb..8fb3b45d49 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -156,7 +156,7 @@ def update(self, module=None): module, "modules", modules_repo=modules_repo, installed_sha=current_version ) else: - version = modules_repo.get_latest_component_version(module, "modules") + version = modules_repo.get_latest_component_version(module, self.component_type) if current_version is not None and not self.force: if current_version == version: From 1f32f7625348f218f96475578b6b1e588a34c484 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 2 Nov 2022 10:51:18 +0100 Subject: [PATCH 424/854] move verify_sha to modules_repo as suggested by @awgymer --- nf_core/components/components_install.py | 22 ---------------------- nf_core/modules/install.py | 2 +- nf_core/modules/modules_repo.py | 20 ++++++++++++++++++++ nf_core/subworkflows/install.py | 2 +- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 18e30e4e42..9cd40005d2 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -16,28 +16,6 @@ log = logging.getLogger(__name__) -def verify_sha(modules_repo, prompt, sha): - """ - Verify that 'sha' and 'prompt' arguments are not provided together. - Verify that the provided SHA exists in the repo. - - Arguments: - modules_repo (ModulesRepo): ModulesRepo object - prompt (bool): prompt asking for SHA - sha (str): provided sha - """ - if prompt and sha is not None: - log.error("Cannot use '--sha' and '--prompt' at the same time!") - return False - - if sha: - if not modules_repo.sha_exists_on_branch(sha): - log.error(f"Commit SHA '{sha}' doesn't exist in '{modules_repo.remote_url}'") - return False - - return True - - def collect_and_verify_name(component_type, component, modules_repo): """ Collect component name. diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 41afc6b8b3..e928eb4c79 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -46,7 +46,7 @@ def install(self, module, silent=False): modules_json.check_up_to_date() # Verify SHA - if not nf_core.components.components_install.verify_sha(self.modules_repo, self.prompt, self.sha): + if not self.modules_repo.verify_sha(self.prompt, self.sha): return False # Check and verify module name diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 2febcac25d..1c242b4764 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -142,6 +142,26 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.avail_module_names = None + def verify_sha(self, prompt, sha): + """ + Verify that 'sha' and 'prompt' arguments are not provided together. + Verify that the provided SHA exists in the repo. + + Arguments: + prompt (bool): prompt asking for SHA + sha (str): provided sha + """ + if prompt and sha is not None: + log.error("Cannot use '--sha' and '--prompt' at the same time!") + return False + + if sha: + if not self.sha_exists_on_branch(sha): + log.error(f"Commit SHA '{sha}' doesn't exist in '{self.remote_url}'") + return False + + return True + def setup_local_repo(self, remote, branch, hide_progress=True): """ Sets up the local git repository. If the repository has been cloned previously, it diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index df0ea3f75a..a5f0c63dcb 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -46,7 +46,7 @@ def install(self, subworkflow, silent=False): modules_json.check_up_to_date() # Verify SHA - if not nf_core.components.components_install.verify_sha(self.modules_repo, self.prompt, self.sha): + if not self.modules_repo.verify_sha(self.prompt, self.sha): return False # Check and verify subworkflow name From ae7fae13133af8153be545d2b10f872600b00fce Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Wed, 2 Nov 2022 09:49:57 -0500 Subject: [PATCH 425/854] Remove redundant extensions --- .devcontainer/devcontainer.json | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 31f3628849..0e022e3578 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,14 +25,7 @@ "extensions": [ "ms-python.python", "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack", - "codezombiech.gitignore", - "esbenp.prettier-vscode", - "EditorConfig.EditorConfig", - "Gruntfuggly.todo-tree", - "mechatroner.rainbow-csv", - "derwat.indent-rainbow", - "streetsidesoftware.code-spell-checker" + "nf-core.nf-core-extensionpack" ] } } From c6fb54975e7401caffc1f258d0516a3cc16706c2 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Wed, 2 Nov 2022 09:50:32 -0500 Subject: [PATCH 426/854] Remove redundant extensions --- .../pipeline-template/.devcontainer/devcontainer.json | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/.devcontainer/devcontainer.json b/nf_core/pipeline-template/.devcontainer/devcontainer.json index d4c8fa8fa5..43e2a18991 100644 --- a/nf_core/pipeline-template/.devcontainer/devcontainer.json +++ b/nf_core/pipeline-template/.devcontainer/devcontainer.json @@ -24,14 +24,7 @@ "extensions": [ "ms-python.python", "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack", - "codezombiech.gitignore", - "esbenp.prettier-vscode", - "EditorConfig.EditorConfig", - "Gruntfuggly.todo-tree", - "mechatroner.rainbow-csv", - "derwat.indent-rainbow", - "streetsidesoftware.code-spell-checker" + "nf-core.nf-core-extensionpack" ] } } From e604cb7afe9727a1331a2adc9143ecd22fe4e335 Mon Sep 17 00:00:00 2001 From: Venkat Malladi Date: Wed, 2 Nov 2022 09:53:03 -0500 Subject: [PATCH 427/854] Update CONTRIBUTING.md --- .github/CONTRIBUTING.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b3c22ef484..27689bf22a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -128,6 +128,15 @@ nf-core create -n testpipeline -d "This pipeline is for testing" nf-core lint nf-core-testpipeline ``` +# To get started: + +- Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) +- Tools installed + - nf-core + - nextflow + + +**Note:** ## Codespaces This repo includes a devcontainer configuration which will create a GitHub Codespace for Nextflow development! @@ -136,10 +145,3 @@ Devcontainer specs: - [DevContainer config](.devcontainer/devcontainer.json) - [Dockerfile](.devcontainer/Dockerfile) - -# To get started: - -- Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) -- Tools installed - - nf-core - - nextflow From 27775a82bbe3e462a364b11e21209213979beb84 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 11:53:05 +0100 Subject: [PATCH 428/854] add get_all_components to modules_json --- nf_core/modules/modules_json.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index a674543124..2e074ebbfd 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -782,6 +782,25 @@ def get_all_modules(self): return self.pipeline_modules + def get_all_components(self, component_type): + """ + Retrieves all pipeline modules/subworkflows that are reported in the modules.json + + Returns: + (dict[str, [(str, str)]]): Dictionary indexed with the repo urls, with a + list of tuples (component_dir, components) as values + """ + if self.modules_json is None: + self.load() + if self.pipeline_components is None: + self.pipeline_components = {} + for repo, repo_entry in self.modules_json.get("repos", {}).items(): + if "modules" in repo_entry: + for dir, modules in repo_entry[component_type].items(): + self.pipeline_components[repo] = [(dir, m) for m in modules] + + return self.pipeline_modules + def get_module_branch(self, module, repo_url, install_dir): """ Gets the branch from which the module was installed From fd9e1d66739f57ec9e3c8f2049cbe7aa9a264b3d Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 12:10:13 +0100 Subject: [PATCH 429/854] first running version of new list command for subworkflows --- nf_core/__main__.py | 64 ++++++++++ nf_core/components/components_command.py | 37 ++++++ nf_core/components/list.py | 150 +++++++++++++++++++++++ nf_core/subworkflows/__init__.py | 1 + nf_core/subworkflows/list.py | 20 +++ 5 files changed, 272 insertions(+) create mode 100644 nf_core/components/list.py create mode 100644 nf_core/subworkflows/list.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a15369485b..5290dac0ff 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -999,6 +999,70 @@ def install(ctx, subworkflow, dir, prompt, force, sha): sys.exit(1) +# nf-core modules list subcommands +@subworkflows.group() +@click.pass_context +def list(ctx): + """ + List modules in a local pipeline or remote repository. + """ + pass + + +# nf-core modules list remote +@list.command() +@click.pass_context +@click.argument("keywords", required=False, nargs=-1, metavar="") +@click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") +def remote(ctx, keywords, json): + """ + List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. + """ + try: + subworkflow_list = nf_core.subworkflows.SubworkflowList( + None, + True, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + + stdout.print(subworkflow_list.list_components(keywords, json)) + except (UserWarning, LookupError) as e: + log.critical(e) + sys.exit(1) + + +# nf-core modules list local +@list.command() +@click.pass_context +@click.argument("keywords", required=False, nargs=-1, metavar="") +@click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: Current working directory][/]", +) +def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin + """ + List subworkflows installed locally in a pipeline + """ + try: + subworkflow_list = nf_core.subworkflows.SubworkflowList( + dir, + False, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + stdout.print(subworkflow_list.list_components(keywords, json)) + except (UserWarning, LookupError) as e: + log.error(e) + sys.exit(1) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 6eabb960f3..d89fb4652e 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -157,3 +157,40 @@ def load_lint_config(self): self.lint_config = yaml.safe_load(fh) except FileNotFoundError: log.debug(f"No lint config file found: {config_fn}") + + def check_component_structure(self, component_name): + """ + Check that the structure of the modules/subworkflow directory in a pipeline is the correct one: + modules/nf-core/TOOL/SUBTOOL | subworkflows/nf-core/SUBWORKFLOW + + Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: + modules/nf-core/modules/TOOL/SUBTOOL + """ + if self.repo_type == "pipeline": + wrong_location_modules = [] + for directory, _, files in os.walk(Path(self.dir, component_name)): + if "main.nf" in files: + module_path = Path(directory).relative_to(Path(self.dir, component_name)) + parts = module_path.parts + # Check that there are modules installed directly under the 'modules' directory + if parts[1] == component_name: + wrong_location_modules.append(module_path) + # If there are modules installed in the wrong location + if len(wrong_location_modules) > 0: + log.info("The modules folder structure is outdated. Reinstalling modules.") + # Remove the local copy of the modules repository + log.info(f"Updating '{self.modules_repo.local_repo_dir}'") + self.modules_repo.setup_local_repo( + self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress + ) + # Move wrong modules to the right directory + for module in wrong_location_modules: + modules_dir = Path(component_name).resolve() + correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) + wrong_dir = Path(modules_dir, module) + shutil.move(wrong_dir, correct_dir) + log.info(f"Moved {wrong_dir} to {correct_dir}.") + shutil.rmtree(Path(self.dir, component_name, self.modules_repo.repo_path, component_name)) + # Regenerate modules.json file + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() diff --git a/nf_core/components/list.py b/nf_core/components/list.py new file mode 100644 index 0000000000..6d91e41f80 --- /dev/null +++ b/nf_core/components/list.py @@ -0,0 +1,150 @@ +import json +import logging + +import rich +from nf_core.components.components_command import ComponentCommand + +import nf_core.modules.modules_utils + +# from .modules_command import ModulesRepo +from nf_core.modules.modules_repo import ModulesRepo + +from nf_core.modules.modules_json import ModulesJson + +log = logging.getLogger(__name__) + + +class ComponentList(ComponentCommand): + def __init__(self, component_type, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False): + super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.remote = remote + + def list_components(self, keywords=None, print_json=False): + """ + Get available modules/subworkflows names from GitHub tree for repo + and print as list to stdout + """ + # Check modules directory structure + # self.check_component_structure(self.component_type) + + # Initialise rich table + table = rich.table.Table() + table.add_column(f"{self.component_type[:-1].capitalize()} Name") + components = [] + + if keywords is None: + keywords = [] + + def pattern_msg(keywords): + if len(keywords) == 0: + return "" + if len(keywords) == 1: + return f" matching pattern '{keywords[0]}'" + else: + quoted_keywords = (f"'{key}'" for key in keywords) + return f" matching patterns {', '.join(quoted_keywords)}" + + # No pipeline given - show all remote + if self.remote: + # Filter the modules/subworkflows by keywords + components = [ + comp + for comp in self.modules_repo.get_avail_components(self.component_type) + if all(k in comp for k in keywords) + ] + + # Nothing found + if len(components) == 0: + log.info( + f"No available {self.component_type} found in {self.modules_repo.remote_url} ({self.modules_repo.branch})" + f"{pattern_msg(keywords)}" + ) + return "" + + for comp in sorted(components): + table.add_row(comp) + + # We have a pipeline - list what's installed + else: + # Check that we are in a pipeline directory + try: + _, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir) + if repo_type != "pipeline": + raise UserWarning( + f"The command 'nf-core {self.component_type} list local' must be run from a pipeline directory.", + ) + except UserWarning as e: + log.error(e) + return "" + # Check whether pipelines is valid + try: + self.has_valid_directory() + except UserWarning as e: + log.error(e) + return "" + + # Verify that 'modules.json' is consistent with the installed modules + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() + + import ipdb + + ipdb.set_trace() + + # Filter by keywords + repos_with_comps = { + repo_url: [comp for comp in components if all(k in comp[1] for k in keywords)] + for repo_url, components in modules_json.get_all_components(self.component_type).items() + } + + # Nothing found + if sum(map(len, repos_with_comps)) == 0: + log.info(f"No nf-core {self.component_type} found in '{self.dir}'{pattern_msg(keywords)}") + return "" + + table.add_column("Repository") + table.add_column("Version SHA") + table.add_column("Message") + table.add_column("Date") + + # Load 'modules.json' + modules_json = modules_json.modules_json + + for repo_url, module_with_dir in sorted(repos_with_comps.items()): + repo_entry = modules_json["repos"].get(repo_url, {}) + for install_dir, component in sorted(module_with_dir): + repo_modules = repo_entry.get(self.component_type) + component_entry = repo_modules.get(install_dir).get(component) + + if component_entry: + version_sha = component_entry["git_sha"] + try: + # pass repo_name to get info on modules even outside nf-core/modules + message, date = ModulesRepo( + remote_url=repo_url, + branch=component_entry["branch"], + ).get_commit_info(version_sha) + except LookupError as e: + log.warning(e) + date = "[red]Not Available" + message = "[red]Not Available" + else: + log.warning( + f"Commit SHA for {self.component_type[:-1]} '{install_dir}/{self.component_type}' is missing from 'modules.json'" + ) + version_sha = "[red]Not Available" + date = "[red]Not Available" + message = "[red]Not Available" + table.add_row(self.component_type, repo_url, version_sha, message, date) + + if print_json: + return json.dumps(components, sort_keys=True, indent=4) + + if self.remote: + log.info( + f"{self.component_type.capitalize()} available from {self.modules_repo.remote_url} ({self.modules_repo.branch})" + f"{pattern_msg(keywords)}:\n" + ) + else: + log.info(f"{self.component_type.capitalize()} installed in '{self.dir}'{pattern_msg(keywords)}:\n") + return table diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index a57bd9686c..6dcab662e4 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,4 +1,5 @@ from .create import SubworkflowCreate from .install import SubworkflowInstall from .subworkflows_test import SubworkflowsTest +from .list import SubworkflowList from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/list.py b/nf_core/subworkflows/list.py new file mode 100644 index 0000000000..6fc1c7c3b6 --- /dev/null +++ b/nf_core/subworkflows/list.py @@ -0,0 +1,20 @@ +import json +import logging + +import rich + +import nf_core.modules.modules_utils + +# from .modules_command import ModulesRepo +from nf_core.modules.modules_repo import ModulesRepo + +from .subworkflows_command import SubworkflowCommand +from nf_core.components.list import ComponentList + + +log = logging.getLogger(__name__) + + +class SubworkflowList(ComponentList): + def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False): + super().__init__("subworkflows", pipeline_dir, remote, remote_url, branch, no_pull) From 1dfafa7016a29dc95d162db522fddb19f0244315 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 12:30:48 +0100 Subject: [PATCH 430/854] first running version of components based list --- nf_core/__main__.py | 4 +- nf_core/components/list.py | 9 +-- nf_core/modules/list.py | 128 +------------------------------ nf_core/modules/modules_json.py | 23 ++++-- nf_core/subworkflows/__init__.py | 2 +- nf_core/subworkflows/list.py | 10 --- tests/modules/list.py | 10 +-- 7 files changed, 27 insertions(+), 159 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 5290dac0ff..0e70f26adb 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -454,7 +454,7 @@ def remote(ctx, keywords, json): ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - stdout.print(module_list.list_modules(keywords, json)) + stdout.print(module_list.list_components(keywords, json)) except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -484,7 +484,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - stdout.print(module_list.list_modules(keywords, json)) + stdout.print(module_list.list_components(keywords, json)) except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index 6d91e41f80..58d4acb985 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -2,15 +2,14 @@ import logging import rich -from nf_core.components.components_command import ComponentCommand import nf_core.modules.modules_utils +from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson # from .modules_command import ModulesRepo from nf_core.modules.modules_repo import ModulesRepo -from nf_core.modules.modules_json import ModulesJson - log = logging.getLogger(__name__) @@ -87,10 +86,6 @@ def pattern_msg(keywords): modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - import ipdb - - ipdb.set_trace() - # Filter by keywords repos_with_comps = { repo_url: [comp for comp in components if all(k in comp[1] for k in keywords)] diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index ea9c69c4a6..e2d5398f9e 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -14,130 +14,4 @@ class ModuleList(ModuleCommand): def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False): - super().__init__(pipeline_dir, remote_url, branch, no_pull) - self.remote = remote - - def list_modules(self, keywords=None, print_json=False): - """ - Get available module names from GitHub tree for repo - and print as list to stdout - """ - # Check modules directory structure - self.check_modules_structure() - - # Initialise rich table - table = rich.table.Table() - table.add_column("Module Name") - modules = [] - - if keywords is None: - keywords = [] - - def pattern_msg(keywords): - if len(keywords) == 0: - return "" - if len(keywords) == 1: - return f" matching pattern '{keywords[0]}'" - else: - quoted_keywords = (f"'{key}'" for key in keywords) - return f" matching patterns {', '.join(quoted_keywords)}" - - # No pipeline given - show all remote - if self.remote: - - # Filter the modules by keywords - modules = [ - mod - for mod in self.modules_repo.get_avail_components(self.component_type) - if all(k in mod for k in keywords) - ] - - # Nothing found - if len(modules) == 0: - log.info( - f"No available modules found in {self.modules_repo.remote_url} ({self.modules_repo.branch})" - f"{pattern_msg(keywords)}" - ) - return "" - - for mod in sorted(modules): - table.add_row(mod) - - # We have a pipeline - list what's installed - else: - # Check that we are in a pipeline directory - try: - _, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir) - if repo_type != "pipeline": - raise UserWarning( - "The command 'nf-core modules list local' must be run from a pipeline directory.", - ) - except UserWarning as e: - log.error(e) - return "" - # Check whether pipelines is valid - try: - self.has_valid_directory() - except UserWarning as e: - log.error(e) - return "" - - # Verify that 'modules.json' is consistent with the installed modules - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - - # Filter by keywords - repos_with_mods = { - repo_url: [mod for mod in modules if all(k in mod[1] for k in keywords)] - for repo_url, modules in modules_json.get_all_modules().items() - } - - # Nothing found - if sum(map(len, repos_with_mods)) == 0: - log.info(f"No nf-core modules found in '{self.dir}'{pattern_msg(keywords)}") - return "" - - table.add_column("Repository") - table.add_column("Version SHA") - table.add_column("Message") - table.add_column("Date") - - # Load 'modules.json' - modules_json = modules_json.modules_json - - for repo_url, module_with_dir in sorted(repos_with_mods.items()): - repo_entry = modules_json["repos"].get(repo_url, {}) - for install_dir, module in sorted(module_with_dir): - repo_modules = repo_entry.get("modules") - module_entry = repo_modules.get(install_dir).get(module) - - if module_entry: - version_sha = module_entry["git_sha"] - try: - # pass repo_name to get info on modules even outside nf-core/modules - message, date = ModulesRepo( - remote_url=repo_url, - branch=module_entry["branch"], - ).get_commit_info(version_sha) - except LookupError as e: - log.warning(e) - date = "[red]Not Available" - message = "[red]Not Available" - else: - log.warning(f"Commit SHA for module '{install_dir}/{module}' is missing from 'modules.json'") - version_sha = "[red]Not Available" - date = "[red]Not Available" - message = "[red]Not Available" - table.add_row(module, repo_url, version_sha, message, date) - - if print_json: - return json.dumps(modules, sort_keys=True, indent=4) - - if self.remote: - log.info( - f"Modules available from {self.modules_repo.remote_url} ({self.modules_repo.branch})" - f"{pattern_msg(keywords)}:\n" - ) - else: - log.info(f"Modules installed in '{self.dir}'{pattern_msg(keywords)}:\n") - return table + super().__init__("modules", pipeline_dir, remote, remote_url, branch, no_pull) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 96a62dd2ef..cb29945a32 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -38,6 +38,7 @@ def __init__(self, pipeline_dir): self.modules_json = None self.pipeline_modules = None self.pipeline_subworkflows = None + self.pipeline_components = None def create(self): """ @@ -329,15 +330,15 @@ def move_module_to_local(self, module, repo_name): to_name += f"-{datetime.datetime.now().strftime('%y%m%d%H%M%S')}" shutil.move(current_path, local_modules_dir / to_name) - def unsynced_modules(self): + def unsynced_components(self): """ - Compute the difference between the modules in the directory and the - modules in the 'modules.json' file. This is done by looking at all + Compute the difference between the modules/subworkflows in the directory and the + modules/subworkflows in the 'modules.json' file. This is done by looking at all directories containing a 'main.nf' file Returns: (untrack_dirs ([ Path ]), missing_installation (dict)): Directories that are not tracked - by the modules.json file, and modules in the modules.json where + by the modules.json file, and modules/subworkflows in the modules.json where the installation directory is missing """ # Add all modules from modules.json to missing_installation @@ -392,6 +393,10 @@ def parse_dirs(self, dirs, missing_installation, component_type): if len(module_repo[component_type][install_dir]) == 0: # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation missing_installation.pop(git_url) + import ipdb + + ipdb.set_trace() + return untracked_dirs, missing_installation def has_git_url_and_modules(self): @@ -479,7 +484,7 @@ def check_up_to_date(self): modules_missing_from_modules_json, subworkflows_missing_from_modules_json, missing_installation, - ) = self.unsynced_modules() + ) = self.unsynced_components() # If there are any modules/subworkflows left in 'modules.json' after all installed are removed, # we try to reinstall them @@ -799,9 +804,9 @@ def get_all_components(self, component_type): for repo, repo_entry in self.modules_json.get("repos", {}).items(): if "modules" in repo_entry: for dir, modules in repo_entry[component_type].items(): - self.pipeline_components[repo] = [(dir, m) for m in modules] + self.pipeline_components[component_type][repo] = [(dir, m) for m in modules] - return self.pipeline_modules + return self.pipeline_components def get_module_branch(self, module, repo_url, install_dir): """ @@ -908,6 +913,10 @@ def resolve_missing_installation(self, missing_installation, component_type): def resolve_missing_from_modules_json(self, missing_from_modules_json, component_type): format_missing = [f"'{dir}'" for dir in missing_from_modules_json] if len(format_missing) == 1: + import ipdb + + ipdb.set_trace() + log.info( f"Recomputing commit SHA for {component_type[:-1]} {format_missing[0]} which was missing from 'modules.json'" ) diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index 6dcab662e4..f10c850d8d 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,5 +1,5 @@ from .create import SubworkflowCreate from .install import SubworkflowInstall -from .subworkflows_test import SubworkflowsTest from .list import SubworkflowList +from .subworkflows_test import SubworkflowsTest from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/list.py b/nf_core/subworkflows/list.py index 6fc1c7c3b6..ddf144ee00 100644 --- a/nf_core/subworkflows/list.py +++ b/nf_core/subworkflows/list.py @@ -1,17 +1,7 @@ -import json import logging -import rich - -import nf_core.modules.modules_utils - -# from .modules_command import ModulesRepo -from nf_core.modules.modules_repo import ModulesRepo - -from .subworkflows_command import SubworkflowCommand from nf_core.components.list import ComponentList - log = logging.getLogger(__name__) diff --git a/tests/modules/list.py b/tests/modules/list.py index 1f2b39abf9..2cd1333faf 100644 --- a/tests/modules/list.py +++ b/tests/modules/list.py @@ -8,7 +8,7 @@ def test_modules_list_remote(self): """Test listing available modules""" mods_list = nf_core.modules.ModuleList(None, remote=True) - listed_mods = mods_list.list_modules() + listed_mods = mods_list.list_components() console = Console(record=True) console.print(listed_mods) output = console.export_text() @@ -18,7 +18,7 @@ def test_modules_list_remote(self): def test_modules_list_remote_gitlab(self): """Test listing the modules in the remote gitlab repo""" mods_list = nf_core.modules.ModuleList(None, remote=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH) - listed_mods = mods_list.list_modules() + listed_mods = mods_list.list_components() print(f"listed modules are {listed_mods}") console = Console(record=True) console.print(listed_mods) @@ -29,7 +29,7 @@ def test_modules_list_remote_gitlab(self): def test_modules_list_pipeline(self): """Test listing locally installed modules""" mods_list = nf_core.modules.ModuleList(self.pipeline_dir, remote=False) - listed_mods = mods_list.list_modules() + listed_mods = mods_list.list_components() console = Console(record=True) console.print(listed_mods) output = console.export_text() @@ -41,7 +41,7 @@ def test_modules_install_and_list_pipeline(self): """Test listing locally installed modules""" self.mods_install.install("trimgalore") mods_list = nf_core.modules.ModuleList(self.pipeline_dir, remote=False) - listed_mods = mods_list.list_modules() + listed_mods = mods_list.list_components() console = Console(record=True) console.print(listed_mods) output = console.export_text() @@ -52,7 +52,7 @@ def test_modules_install_gitlab_and_list_pipeline(self): """Test listing locally installed modules""" self.mods_install_gitlab.install("fastqc") mods_list = nf_core.modules.ModuleList(self.pipeline_dir, remote=False) - listed_mods = mods_list.list_modules() + listed_mods = mods_list.list_components() console = Console(record=True) console.print(listed_mods) output = console.export_text() From efba8908fe56edbfd31eac4bc71fbcbfc2e80f9d Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 11:53:05 +0100 Subject: [PATCH 431/854] add get_all_components to modules_json --- nf_core/modules/modules_json.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index b1f1ab3ca8..96a62dd2ef 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -784,6 +784,25 @@ def get_all_modules(self): return self.pipeline_modules + def get_all_components(self, component_type): + """ + Retrieves all pipeline modules/subworkflows that are reported in the modules.json + + Returns: + (dict[str, [(str, str)]]): Dictionary indexed with the repo urls, with a + list of tuples (component_dir, components) as values + """ + if self.modules_json is None: + self.load() + if self.pipeline_components is None: + self.pipeline_components = {} + for repo, repo_entry in self.modules_json.get("repos", {}).items(): + if "modules" in repo_entry: + for dir, modules in repo_entry[component_type].items(): + self.pipeline_components[repo] = [(dir, m) for m in modules] + + return self.pipeline_modules + def get_module_branch(self, module, repo_url, install_dir): """ Gets the branch from which the module was installed From d50d6a18d8aa9f78d4e9a8839c0863e4718494b5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 12:05:05 +0100 Subject: [PATCH 432/854] small fixes for sw install --- nf_core/modules/modules_repo.py | 3 ++- nf_core/subworkflows/install.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 1c242b4764..d0a831c933 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -401,6 +401,7 @@ def get_component_git_log(self, component_name, component_type, depth=None): commits_new = [ {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_new ] + commits_old = [] if component_type == "modules": # Grab commits also from previous modules structure component_path = os.path.join("modules", component_name) @@ -408,7 +409,7 @@ def get_component_git_log(self, component_name, component_type, depth=None): commits_old = [ {"git_sha": commit.hexsha, "trunc_message": commit.message.partition("\n")[0]} for commit in commits_old ] - commits = iter(commits_new + commits_old) + commits = iter(commits_new + commits_old) return commits def get_latest_component_version(self, component_name, component_type): diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index a5f0c63dcb..439cf187b7 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -91,7 +91,7 @@ def install(self, subworkflow, silent=False): log.debug(f"Installing subworkflow '{subworkflow}' at hash {version} from {self.modules_repo.remote_url}") # Download subworkflow files - if not self.install_subworkflow_files(subworkflow, version, self.modules_repo, install_folder): + if not self.install_component_files(subworkflow, version, self.modules_repo, install_folder): return False # Install included modules and subworkflows From cf5f0553120886d0764da1d77d607f0e1aae8dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Langer?= <61791748+bjlang@users.noreply.github.com> Date: Thu, 3 Nov 2022 12:39:23 +0100 Subject: [PATCH 433/854] Fix PythonBlack --- nf_core/subworkflows/info.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index 042cb9f0e8..5bfbe54794 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -281,7 +281,9 @@ def generate_subworkflow_info_help(self): if self.remote_location != NF_CORE_MODULES_REMOTE: cmd_base = f"nf-core subworkflows --git-remote {self.remote_location}" renderables.append( - Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.subworkflow}\n") + Text.from_markup( + f"\n :computer: Installation command: [magenta]{cmd_base} install {self.subworkflow}\n" + ) ) return Group(*renderables) From abbd1bc86623e9966b586c0dae2bf9097ecc1ffb Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 12:58:10 +0100 Subject: [PATCH 434/854] fix for modules list --- nf_core/components/list.py | 7 ++++--- nf_core/modules/list.py | 11 ++--------- nf_core/modules/modules_json.py | 16 +++++----------- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index 58d4acb985..3ff03a738d 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -66,6 +66,7 @@ def pattern_msg(keywords): # We have a pipeline - list what's installed else: # Check that we are in a pipeline directory + try: _, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir) if repo_type != "pipeline": @@ -105,9 +106,9 @@ def pattern_msg(keywords): # Load 'modules.json' modules_json = modules_json.modules_json - for repo_url, module_with_dir in sorted(repos_with_comps.items()): + for repo_url, component_with_dir in sorted(repos_with_comps.items()): repo_entry = modules_json["repos"].get(repo_url, {}) - for install_dir, component in sorted(module_with_dir): + for install_dir, component in sorted(component_with_dir): repo_modules = repo_entry.get(self.component_type) component_entry = repo_modules.get(install_dir).get(component) @@ -130,7 +131,7 @@ def pattern_msg(keywords): version_sha = "[red]Not Available" date = "[red]Not Available" message = "[red]Not Available" - table.add_row(self.component_type, repo_url, version_sha, message, date) + table.add_row(component, repo_url, version_sha, message, date) if print_json: return json.dumps(components, sort_keys=True, indent=4) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index e2d5398f9e..c7dc943f9e 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -1,17 +1,10 @@ -import json import logging -import rich - -import nf_core.modules.modules_utils - -from .modules_command import ModuleCommand -from .modules_json import ModulesJson -from .modules_repo import ModulesRepo +from nf_core.components.list import ComponentList log = logging.getLogger(__name__) -class ModuleList(ModuleCommand): +class ModuleList(ComponentList): def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False): super().__init__("modules", pipeline_dir, remote, remote_url, branch, no_pull) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cb29945a32..17282a3620 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -381,6 +381,7 @@ def parse_dirs(self, dirs, missing_installation, component_type): if not component_in_file: # If it is not, add it to the list of missing subworkflow untracked_dirs.append(component) + else: # If it does, remove the subworkflow from missing_installation module_repo = missing_installation[git_url] @@ -393,9 +394,6 @@ def parse_dirs(self, dirs, missing_installation, component_type): if len(module_repo[component_type][install_dir]) == 0: # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation missing_installation.pop(git_url) - import ipdb - - ipdb.set_trace() return untracked_dirs, missing_installation @@ -800,11 +798,11 @@ def get_all_components(self, component_type): if self.modules_json is None: self.load() if self.pipeline_components is None: - self.pipeline_components = {} + self.pipeline_components = {component_type: {}} for repo, repo_entry in self.modules_json.get("repos", {}).items(): - if "modules" in repo_entry: - for dir, modules in repo_entry[component_type].items(): - self.pipeline_components[component_type][repo] = [(dir, m) for m in modules] + if component_type in repo_entry: + for dir, components in repo_entry[component_type].items(): + self.pipeline_components[repo] = [(dir, m) for m in components] return self.pipeline_components @@ -913,10 +911,6 @@ def resolve_missing_installation(self, missing_installation, component_type): def resolve_missing_from_modules_json(self, missing_from_modules_json, component_type): format_missing = [f"'{dir}'" for dir in missing_from_modules_json] if len(format_missing) == 1: - import ipdb - - ipdb.set_trace() - log.info( f"Recomputing commit SHA for {component_type[:-1]} {format_missing[0]} which was missing from 'modules.json'" ) From cf63ca00daf7d3045c59e1a364f5b895db8580b8 Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Thu, 3 Nov 2022 12:58:53 +0100 Subject: [PATCH 435/854] change module_utils.py to modules_utils.py --- nf_core/subworkflows/info.py | 6 +++--- nf_core/subworkflows/list.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index 5bfbe54794..6fc0b54e86 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -12,8 +12,8 @@ from rich.table import Table from rich.text import Text -import nf_core.modules.module_utils -from nf_core.modules.module_utils import get_repo_type +import nf_core.modules.modules_utils +from nf_core.modules.modules_utils import get_repo_type from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE, ModulesRepo @@ -61,7 +61,7 @@ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): self.modules_repo = ModulesRepo(remote_url, branch, no_pull) try: if self.dir: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = get_repo_type(self.dir) else: self.repo_type = None except LookupError as e: diff --git a/nf_core/subworkflows/list.py b/nf_core/subworkflows/list.py index 71eff5fdfd..ebc5e57222 100644 --- a/nf_core/subworkflows/list.py +++ b/nf_core/subworkflows/list.py @@ -4,7 +4,7 @@ import rich -import nf_core.modules.module_utils +import nf_core.modules.modules_utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, ModulesRepo @@ -19,7 +19,7 @@ def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_p self.modules_repo = ModulesRepo(remote_url, branch, no_pull) try: if self.dir: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir) else: self.repo_type = None except LookupError as e: @@ -71,7 +71,7 @@ def pattern_msg(keywords): else: # Check that we are in a pipeline directory try: - _, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + _, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir) if repo_type != "pipeline": raise UserWarning( "The command 'nf-core subworkflows list local' must be run from a pipeline directory.", From 30905319fbed33686e9632f9735295dbdd1048f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 3 Nov 2022 13:44:05 +0100 Subject: [PATCH 436/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/__main__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0e70f26adb..86ec88e1ee 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -999,7 +999,7 @@ def install(ctx, subworkflow, dir, prompt, force, sha): sys.exit(1) -# nf-core modules list subcommands +# nf-core subworkflows list subcommands @subworkflows.group() @click.pass_context def list(ctx): @@ -1009,14 +1009,14 @@ def list(ctx): pass -# nf-core modules list remote +# nf-core subworkflows list remote @list.command() @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") def remote(ctx, keywords, json): """ - List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. + List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ try: subworkflow_list = nf_core.subworkflows.SubworkflowList( @@ -1033,7 +1033,7 @@ def remote(ctx, keywords, json): sys.exit(1) -# nf-core modules list local +# nf-core subworkflows list local @list.command() @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") From 06a272de426dd95f857a0a0374483c9e8588a5c6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 13:46:44 +0100 Subject: [PATCH 437/854] remove implementation relicts --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 17282a3620..1b8ea36687 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -798,7 +798,7 @@ def get_all_components(self, component_type): if self.modules_json is None: self.load() if self.pipeline_components is None: - self.pipeline_components = {component_type: {}} + self.pipeline_components = {} for repo, repo_entry in self.modules_json.get("repos", {}).items(): if component_type in repo_entry: for dir, components in repo_entry[component_type].items(): From 3d0051782b61563145f6c68b238071336a101e04 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 3 Nov 2022 13:49:44 +0100 Subject: [PATCH 438/854] add tests for subworkflows install command --- nf_core/components/components_install.py | 15 ++--- nf_core/modules/install.py | 2 +- nf_core/modules/modules_json.py | 10 +-- nf_core/modules/patch.py | 2 +- nf_core/modules/update.py | 48 +++++++++++---- nf_core/subworkflows/install.py | 2 +- tests/modules/install.py | 2 +- tests/modules/update.py | 20 ++++-- tests/subworkflows/install.py | 78 ++++++++++++++++++++++++ tests/test_modules.py | 1 + tests/test_subworkflows.py | 18 +++++- tests/utils.py | 1 + 12 files changed, 167 insertions(+), 32 deletions(-) create mode 100644 tests/subworkflows/install.py diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 9cd40005d2..ed730edeff 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -42,19 +42,20 @@ def collect_and_verify_name(component_type, component, modules_repo): return component -def check_component_installed(component_type, component, current_version, component_dir, modules_repo, force): +def check_component_installed(component_type, component, current_version, component_dir, modules_repo, force, prompt): """ Check that the module/subworkflow is not already installed """ if (current_version is not None and os.path.exists(component_dir)) and not force: log.info(f"{component_type[:-1].title()} is already installed.") - message = "?" if component_type == "modules" else " of this subworkflow and all it's imported modules?" - force = questionary.confirm( - f"{component_type[:-1].title()} {component} is already installed. \nDo you want to force the reinstallation{message}", - style=nf_core.utils.nfcore_question_style, - default=False, - ).unsafe_ask() + if prompt: + message = "?" if component_type == "modules" else " of this subworkflow and all it's imported modules?" + force = questionary.confirm( + f"{component_type[:-1].title()} {component} is already installed. \nDo you want to force the reinstallation{message}", + style=nf_core.utils.nfcore_question_style, + default=False, + ).unsafe_ask() if not force: repo_flag = "" if modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {modules_repo.remote_url} " diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index e928eb4c79..ad93049f56 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -69,7 +69,7 @@ def install(self, module, silent=False): # Check that the module is not already installed if not nf_core.components.components_install.check_component_installed( - self.component_type, module, current_version, module_dir, self.modules_repo, self.force + self.component_type, module, current_version, module_dir, self.modules_repo, self.force, self.prompt ): return False diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 96a62dd2ef..29eb8c3309 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -803,9 +803,9 @@ def get_all_components(self, component_type): return self.pipeline_modules - def get_module_branch(self, module, repo_url, install_dir): + def get_component_branch(self, component_type, component, repo_url, install_dir): """ - Gets the branch from which the module was installed + Gets the branch from which the module/subworkflow was installed Returns: (str): The branch name @@ -817,14 +817,14 @@ def get_module_branch(self, module, repo_url, install_dir): branch = ( self.modules_json["repos"] .get(repo_url, {}) - .get("modules", {}) + .get(component_type, {}) .get(install_dir, {}) - .get(module, {}) + .get(component, {}) .get("branch") ) if branch is None: raise LookupError( - f"Could not find branch information for module '{Path(install_dir, module)}'." + f"Could not find branch information for component '{Path(install_dir, component)}'." f"Please remove the 'modules.json' and rerun the command to recreate it" ) return branch diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 19b3d6a407..5d51b0beb7 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -62,7 +62,7 @@ def patch(self, module=None): f"The '{module_fullname}' module does not have a valid version in the 'modules.json' file. Cannot compute patch" ) # Get the module branch and reset it in the ModulesRepo object - module_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url, module_dir) + module_branch = self.modules_json.get_component_branch(self.component_type, module, self.modules_repo.remote_url, module_dir) if module_branch != self.modules_repo.branch: self.modules_repo.setup_branch(module_branch) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 8fb3b45d49..9e2a829d1d 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -315,7 +315,9 @@ def get_single_module_info(self, module): log.info(f"Updating module to ({sha})") # Check if the update branch is the same as the installation branch - current_branch = self.modules_json.get_module_branch(module, self.modules_repo.remote_url, install_dir) + current_branch = self.modules_json.get_component_branch( + self.component_type, module, self.modules_repo.remote_url, install_dir + ) new_branch = self.modules_repo.branch if current_branch != new_branch: log.warning( @@ -361,11 +363,23 @@ def get_all_modules_info(self, branch=None): for module_dir, module in modules: try: modules_info[repo_name][module_dir].append( - (module, self.sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + ( + module, + self.sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) ) except KeyError: modules_info[repo_name][module_dir] = [ - (module, self.sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + ( + module, + self.sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) ] elif isinstance(self.update_config[repo_name], dict): # If it is a dict, then there are entries for individual modules or module directories @@ -381,7 +395,9 @@ def get_all_modules_info(self, branch=None): ( module, custom_sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), ) ) except KeyError: @@ -389,7 +405,9 @@ def get_all_modules_info(self, branch=None): ( module, custom_sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), ) ] if self.sha is not None: @@ -409,7 +427,9 @@ def get_all_modules_info(self, branch=None): ( module, self.sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), ) ) except KeyError: @@ -417,7 +437,9 @@ def get_all_modules_info(self, branch=None): ( module, self.sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), ) ] elif isinstance(dir_config[module], str): @@ -428,7 +450,9 @@ def get_all_modules_info(self, branch=None): ( module, custom_sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), ) ) except KeyError: @@ -436,7 +460,9 @@ def get_all_modules_info(self, branch=None): ( module, custom_sha, - self.modules_json.get_module_branch(module, repo_name, module_dir), + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), ) ] if self.sha is not None: @@ -455,11 +481,11 @@ def get_all_modules_info(self, branch=None): for module_dir, module in modules: try: modules_info[repo_name][module_dir].append( - (module, custom_sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + (module, custom_sha, self.modules_json.get_component_branch(self.component_type, module, repo_name, module_dir)) ) except KeyError: modules_info[repo_name][module_dir] = [ - (module, custom_sha, self.modules_json.get_module_branch(module, repo_name, module_dir)) + (module, custom_sha, self.modules_json.get_component_branch(self.component_type, module, repo_name, module_dir)) ] if self.sha is not None: overridden_repos.append(repo_name) diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 439cf187b7..aa87a0013b 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -69,7 +69,7 @@ def install(self, subworkflow, silent=False): # Check that the subworkflow is not already installed if not nf_core.components.components_install.check_component_installed( - self.component_type, subworkflow, current_version, subworkflow_dir, self.modules_repo, self.force + self.component_type, subworkflow, current_version, subworkflow_dir, self.modules_repo, self.force, self.prompt ): return False diff --git a/tests/modules/install.py b/tests/modules/install.py index 7b7d86c2d7..8d43d1c588 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -67,4 +67,4 @@ def test_modules_install_different_branch_succeed(self): # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH diff --git a/tests/modules/update.py b/tests/modules/update.py index 9ff098d88e..9ca293add8 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -231,7 +231,10 @@ def test_update_different_branch_single_module(self): # Verify that the branch entry was updated correctly modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_module_branch("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) + == GITLAB_BRANCH_TEST_BRANCH + ) assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA @@ -249,10 +252,16 @@ def test_update_different_branch_mixed_modules_main(self): modules_json = ModulesJson(self.pipeline_dir) # Verify that the branch entry was updated correctly - assert modules_json.get_module_branch("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) + == GITLAB_BRANCH_TEST_BRANCH + ) assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA # MultiQC is present in both branches but should've been updated using the 'main' branch - assert modules_json.get_module_branch("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_DEFAULT_BRANCH + assert ( + modules_json.get_component_branch(self.component_type, "multiqc", GITLAB_URL, GITLAB_REPO) + == GITLAB_DEFAULT_BRANCH + ) def test_update_different_branch_mix_modules_branch_test(self): @@ -272,7 +281,10 @@ def test_update_different_branch_mix_modules_branch_test(self): ) assert update_obj.update() - assert modules_json.get_module_branch("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_component_branch(self.component_type, "multiqc", GITLAB_URL, GITLAB_REPO) + == GITLAB_BRANCH_TEST_BRANCH + ) assert modules_json.get_module_version("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py new file mode 100644 index 0000000000..94b2d79ba0 --- /dev/null +++ b/tests/subworkflows/install.py @@ -0,0 +1,78 @@ +import os + +import pytest + +from nf_core.modules.modules_json import ModulesJson +from nf_core.subworkflows.install import SubworkflowInstall + +from ..utils import ( + GITLAB_BRANCH_TEST_BRANCH, + GITLAB_REPO, + GITLAB_SUBWORKFLOWS_BRANCH, + GITLAB_URL, + with_temporary_folder, +) + + +def test_subworkflow_install_nopipeline(self): + """Test installing a subworkflow - no pipeline given""" + self.subworkflow_install.dir = None + assert self.subworkflow_install.install("foo") is False + + +@with_temporary_folder +def test_subworkflows_install_emptypipeline(self, tmpdir): + """Test installing a subworkflow - empty dir given""" + os.mkdir(os.path.join(tmpdir, "nf-core-pipe")) + self.subworkflow_install.dir = os.path.join(tmpdir, "nf-core-pipe") + with pytest.raises(UserWarning) as excinfo: + self.subworkflow_install.install("foo") + assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) + + +def test_subworkflows_install_nosubworkflow(self): + """Test installing a subworkflow - unrecognised subworkflow given""" + assert self.subworkflow_install.install("foo") is False + + +def test_subworkflows_install_bam_sort_stats_samtools(self): + """Test installing a subworkflow - bam_sort_stats_samtools""" + assert self.subworkflow_install.install("bam_sort_stats_samtools") is not False + subworkflow_path = os.path.join(self.subworkflow_install.dir, "subworkflows", "nf-core", "bam_sort_stats_samtools") + sub_subworkflow_path = os.path.join(self.subworkflow_install.dir, "subworkflows", "nf-core", "bam_stats_samtools") + samtools_index_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") + samtools_sort_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "sort") + samtools_stats_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "stats") + samtools_idxstats_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "idxstats") + samtools_flagstat_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "flagstat") + assert os.path.exists(subworkflow_path) + assert os.path.exists(sub_subworkflow_path) + assert os.path.exists(samtools_index_path) + assert os.path.exists(samtools_sort_path) + assert os.path.exists(samtools_stats_path) + assert os.path.exists(samtools_idxstats_path) + assert os.path.exists(samtools_flagstat_path) + + +def test_subworkflows_install_bam_sort_stats_samtools_twice(self): + """Test installing a subworkflow - bam_sort_stats_samtools already there""" + self.subworkflow_install.install("bam_sort_stats_samtools") + assert self.subworkflow_install.install("bam_sort_stats_samtools") is False + + +def test_subworkflows_install_from_gitlab(self): + """Test installing a subworkflow from GitLab""" + assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True + # Verify that the branch entry was added correctly + modules_json = ModulesJson(self.pipeline_dir) + assert ( + modules_json.get_component_branch(self.component_type, "bam_stats_samtools", GITLAB_URL, GITLAB_REPO) + == GITLAB_SUBWORKFLOWS_BRANCH + ) + + +def test_subworkflows_install_different_branch_fail(self): + """Test installing a subworkflow from a different branch""" + install_obj = SubworkflowInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) + # The bam_stats_samtools subworkflow does not exists in the branch-test branch + assert install_obj.install("bam_stats_samtools") is False diff --git a/tests/test_modules.py b/tests/test_modules.py index e1b9609699..0b68dc9df2 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -51,6 +51,7 @@ class TestModules(unittest.TestCase): def setUp(self): """Create a new PipelineSchema and Launch objects""" self.tmp_dir = tempfile.mkdtemp() + self.component_type = "modules" # Set up the schema root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index ea4fa986dd..dd4539a246 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -13,7 +13,7 @@ import nf_core.modules import nf_core.subworkflows -from .utils import GITLAB_URL, mock_api_calls +from .utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, mock_api_calls def create_modules_repo_dummy(tmp_dir): @@ -46,6 +46,7 @@ class TestSubworkflows(unittest.TestCase): def setUp(self): """Create a new PipelineStructure and Launch objects""" self.tmp_dir = tempfile.mkdtemp() + self.component_type = "subworkflows" # Set up the pipeline structure root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -55,6 +56,12 @@ def setUp(self): "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() + # Set up install objects + self.subworkflow_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=False) + self.subworkflow_install_gitlab = nf_core.subworkflows.SubworkflowInstall( + self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH + ) + # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) @@ -67,6 +74,15 @@ def setUp(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) + from .subworkflows.install import ( + test_subworkflow_install_nopipeline, + test_subworkflows_install_bam_sort_stats_samtools, + test_subworkflows_install_bam_sort_stats_samtools_twice, + test_subworkflows_install_different_branch_fail, + test_subworkflows_install_emptypipeline, + test_subworkflows_install_from_gitlab, + test_subworkflows_install_nosubworkflow, + ) from .subworkflows.subworkflows_test import ( test_subworkflows_test_check_inputs, test_subworkflows_test_no_installed_subworkflows, diff --git a/tests/utils.py b/tests/utils.py index 483f817f20..65c7d48758 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,6 +15,7 @@ GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" GITLAB_REPO = "nf-core" GITLAB_DEFAULT_BRANCH = "main-restructure" +GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows" # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester-restructure" GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" From d3b9f32c6aa5db15391321ed4b2dddb4a2b98cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Langer?= <61791748+bjlang@users.noreply.github.com> Date: Thu, 3 Nov 2022 13:59:06 +0100 Subject: [PATCH 439/854] fix isort --- nf_core/subworkflows/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index 6fc0b54e86..1489364971 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -13,9 +13,9 @@ from rich.text import Text import nf_core.modules.modules_utils -from nf_core.modules.modules_utils import get_repo_type from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE, ModulesRepo +from nf_core.modules.modules_utils import get_repo_type log = logging.getLogger(__name__) From eed9bfe51731414fb569138faef78a08c2135803 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 3 Nov 2022 13:59:18 +0100 Subject: [PATCH 440/854] modify changelog --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61ca21ffed..0428304c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,7 @@ - Fix bug when updating modules from old version in old folder structure - Don't remove local copy of modules repo, only update it with fetch ([#1881](https://github.com/nf-core/tools/pull/1881)) - Add subworkflow commands create-test-yml, create and install ([#1897](https://github.com/nf-core/tools/pull/1897)) -- Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) - Improve test coverage of `sync.py` and `__main__.py` ([#1936](https://github.com/nf-core/tools/pull/1936), [#1965](https://github.com/nf-core/tools/pull/1965)) -- `check_up_to_date()` function from `modules_json` also checks for subworkflows. - The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). - Only warn when checking that the pipeline directory contains a `main.nf` and a `nextflow.config` file if the pipeline is not an nf-core pipeline [#1964](https://github.com/nf-core/tools/pull/1964) - Add file `versions.yml` when generating `test.yml` with `nf-core modules create-test-yml` but don't check for md5sum [#1963](https://github.com/nf-core/tools/pull/1963) @@ -31,6 +29,12 @@ - Update patch file paths if the modules directory has the old structure ([#1878](https://github.com/nf-core/tools/pull/1878)) +### Subworkflows + +- Add tests for subworkflows install command ([#1996](https://github.com/nf-core/tools/pull/1996)) +- `check_up_to_date()` function from `modules_json` also checks for subworkflows. +- Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) + ## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] ### Template From 8473fda117b2edc9744385e09593203d5a9fc565 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 3 Nov 2022 15:12:14 +0000 Subject: [PATCH 441/854] [automated] Fix code linting --- nf_core/modules/patch.py | 4 +++- nf_core/modules/update.py | 16 ++++++++++++++-- nf_core/subworkflows/install.py | 8 +++++++- tests/modules/install.py | 5 ++++- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 5d51b0beb7..84e839a033 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -62,7 +62,9 @@ def patch(self, module=None): f"The '{module_fullname}' module does not have a valid version in the 'modules.json' file. Cannot compute patch" ) # Get the module branch and reset it in the ModulesRepo object - module_branch = self.modules_json.get_component_branch(self.component_type, module, self.modules_repo.remote_url, module_dir) + module_branch = self.modules_json.get_component_branch( + self.component_type, module, self.modules_repo.remote_url, module_dir + ) if module_branch != self.modules_repo.branch: self.modules_repo.setup_branch(module_branch) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 9e2a829d1d..dcdde1c67b 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -481,11 +481,23 @@ def get_all_modules_info(self, branch=None): for module_dir, module in modules: try: modules_info[repo_name][module_dir].append( - (module, custom_sha, self.modules_json.get_component_branch(self.component_type, module, repo_name, module_dir)) + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) ) except KeyError: modules_info[repo_name][module_dir] = [ - (module, custom_sha, self.modules_json.get_component_branch(self.component_type, module, repo_name, module_dir)) + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) ] if self.sha is not None: overridden_repos.append(repo_name) diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index aa87a0013b..494c03c24b 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -69,7 +69,13 @@ def install(self, subworkflow, silent=False): # Check that the subworkflow is not already installed if not nf_core.components.components_install.check_component_installed( - self.component_type, subworkflow, current_version, subworkflow_dir, self.modules_repo, self.force, self.prompt + self.component_type, + subworkflow, + current_version, + subworkflow_dir, + self.modules_repo, + self.force, + self.prompt, ): return False diff --git a/tests/modules/install.py b/tests/modules/install.py index 8d43d1c588..1c3f1aefc7 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -67,4 +67,7 @@ def test_modules_install_different_branch_succeed(self): # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) - assert modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_BRANCH + assert ( + modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) + == GITLAB_BRANCH_TEST_BRANCH + ) From f289a69f508ef36c851ed41ac4f9952bfb59b45b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 17:30:53 +0100 Subject: [PATCH 442/854] add test for sw list --- nf_core/subworkflows/install.py | 6 ++-- tests/subworkflows/create.py | 12 +++++--- tests/subworkflows/list.py | 49 +++++++++++++++++++++++++++++++++ tests/test_modules.py | 1 - tests/test_subworkflows.py | 26 +++++++++++------ tests/utils.py | 1 + 6 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 tests/subworkflows/list.py diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 439cf187b7..d087bd1abd 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -79,8 +79,10 @@ def install(self, subworkflow, silent=False): if not version: return False - # Remove subworkflow if force is set - if self.force: + # Remove subworkflow if force is set and component is installed + if self.force and nf_core.components.components_install.check_component_installed( + self.component_type, subworkflow, current_version, subworkflow_dir, self.modules_repo, self.force + ): log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") self.clear_component_dir(subworkflow, subworkflow_dir) nf_core.components.components_install.clean_modules_json( diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index ac421064e8..eac4929136 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -7,14 +7,18 @@ def test_subworkflows_create_succeed(self): """Succeed at creating a subworkflow from the template inside a pipeline""" - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(self.pipeline_dir, "test_subworkflow", "@author", True) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( + self.pipeline_dir, "test_subworkflow_local", "@author", True + ) subworkflow_create.create() - assert os.path.exists(os.path.join(self.pipeline_dir, "subworkflows", "local", "test_subworkflow.nf")) + assert os.path.exists(os.path.join(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local.nf")) def test_subworkflows_create_fail_exists(self): """Fail at creating the same subworkflow twice""" - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(self.pipeline_dir, "test_subworkflow", "@author", False) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( + self.pipeline_dir, "test_subworkflow2", "@author", False + ) subworkflow_create.create() with pytest.raises(UserWarning) as excinfo: subworkflow_create.create() @@ -24,7 +28,7 @@ def test_subworkflows_create_fail_exists(self): def test_subworkflows_create_nfcore_modules(self): """Create a subworkflow in nf-core/modules clone""" subworkflow_create = nf_core.subworkflows.SubworkflowCreate( - self.nfcore_modules, "test_subworkflow", "@author", False + self.nfcore_modules, "test_subworkflow", "@author", True ) subworkflow_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf")) diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py new file mode 100644 index 0000000000..bbb2f82235 --- /dev/null +++ b/tests/subworkflows/list.py @@ -0,0 +1,49 @@ +from rich.console import Console + +import nf_core.subworkflows + +from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL + + +def test_subworkflows_list_remote(self): + """Test listing available subworkflows""" + subworkflows_list = nf_core.subworkflows.SubworkflowList(None, remote=True) + listed_subworkflows = subworkflows_list.list_components() + console = Console(record=True) + console.print(listed_subworkflows) + output = console.export_text() + assert "bam_stats" in output + + +def test_subworkflows_list_remote_gitlab(self): + """Test listing the subworkflows in the remote gitlab repo""" + subworkflows_list = nf_core.subworkflows.SubworkflowList( + None, remote=True, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH + ) + listed_subworkflows = subworkflows_list.list_components() + console = Console(record=True) + console.print(listed_subworkflows) + output = console.export_text() + assert "bam_stats" in output + + +def test_subworkflows_install_and_list_subworkflows(self): + """Test listing locally installed subworkflows""" + self.sw_install.install("bam_stats_samtools") + subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) + listed_subworkflows = subworkflows_list.list_components() + console = Console(record=True) + console.print(listed_subworkflows) + output = console.export_text() + assert "bam_stats" in output + + +def test_subworkflows_install_gitlab_and_list_subworkflows(self): + """Test listing locally installed subworkflows""" + self.sw_install_gitlab.install("bam_stats_samtools") + subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) + listed_subworkflows = subworkflows_list.list_components() + console = Console(record=True) + console.print(listed_subworkflows) + output = console.export_text() + assert "bam_stats" in output diff --git a/tests/test_modules.py b/tests/test_modules.py index e1b9609699..d52386665c 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Tests covering the modules commands """ diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index ea4fa986dd..fbe33dbe0f 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,9 +1,7 @@ -#!/usr/bin/env python """ Tests covering the subworkflows commands """ import os -import shutil import tempfile import unittest @@ -13,10 +11,10 @@ import nf_core.modules import nf_core.subworkflows -from .utils import GITLAB_URL, mock_api_calls +from .utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, mock_api_calls -def create_modules_repo_dummy(tmp_dir): +def create_subworkflows_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" root_dir = os.path.join(tmp_dir, "modules") @@ -32,10 +30,8 @@ def create_modules_repo_dummy(tmp_dir): fh.writelines(["repository_type: modules", "\n"]) with requests_mock.Mocker() as mock: - mock_api_calls(mock, "bpipe", "0.9.11") - # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) - module_create.create() + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) + subworkflow_create.create() return root_dir @@ -56,7 +52,13 @@ def setUp(self): ).init_pipeline() # Set up the nf-core/modules repo dummy - self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) + self.nfcore_modules = create_subworkflows_repo_dummy(self.tmp_dir) + + # Set up install objects + self.sw_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=False) + self.sw_install_gitlab = nf_core.subworkflows.SubworkflowInstall( + self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH + ) ############################################ # Test of the individual modules commands. # @@ -67,6 +69,12 @@ def setUp(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) + from .subworkflows.list import ( + test_subworkflows_install_and_list_subworkflows, + test_subworkflows_install_gitlab_and_list_subworkflows, + test_subworkflows_list_remote, + test_subworkflows_list_remote_gitlab, + ) from .subworkflows.subworkflows_test import ( test_subworkflows_test_check_inputs, test_subworkflows_test_no_installed_subworkflows, diff --git a/tests/utils.py b/tests/utils.py index 483f817f20..65c7d48758 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,6 +15,7 @@ GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" GITLAB_REPO = "nf-core" GITLAB_DEFAULT_BRANCH = "main-restructure" +GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows" # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester-restructure" GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" From 7a2fa10c17f677ca38462f04521ce5c5177d1c9a Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 3 Nov 2022 17:31:24 +0100 Subject: [PATCH 443/854] use keywords as arrays in list --- nf_core/components/list.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index 3ff03a738d..401eade78a 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -18,7 +18,7 @@ def __init__(self, component_type, pipeline_dir, remote=True, remote_url=None, b super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) self.remote = remote - def list_components(self, keywords=None, print_json=False): + def list_components(self, keywords=[], print_json=False): """ Get available modules/subworkflows names from GitHub tree for repo and print as list to stdout @@ -31,9 +31,6 @@ def list_components(self, keywords=None, print_json=False): table.add_column(f"{self.component_type[:-1].capitalize()} Name") components = [] - if keywords is None: - keywords = [] - def pattern_msg(keywords): if len(keywords) == 0: return "" From 15397775f263194d28eae6858ac26c4b58c454db Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 3 Nov 2022 16:35:45 +0000 Subject: [PATCH 444/854] [automated] Fix code linting --- .devcontainer/devcontainer.json | 6 +----- .github/CONTRIBUTING.md | 2 +- nf_core/pipeline-template/.devcontainer/devcontainer.json | 6 +----- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0e022e3578..b5138b1d0d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -22,11 +22,7 @@ }, // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack" - ] + "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"] } } } diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 27689bf22a..19ccba5566 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -135,8 +135,8 @@ nf-core lint nf-core-testpipeline - nf-core - nextflow - **Note:** + ## Codespaces This repo includes a devcontainer configuration which will create a GitHub Codespace for Nextflow development! diff --git a/nf_core/pipeline-template/.devcontainer/devcontainer.json b/nf_core/pipeline-template/.devcontainer/devcontainer.json index 43e2a18991..ea27a5843a 100644 --- a/nf_core/pipeline-template/.devcontainer/devcontainer.json +++ b/nf_core/pipeline-template/.devcontainer/devcontainer.json @@ -21,11 +21,7 @@ }, // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "nf-core.nf-core-extensionpack" - ] + "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"] } } } From ad2db6ceb4b8b79f939cebb60e71a586e4341a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 3 Nov 2022 17:39:05 +0100 Subject: [PATCH 445/854] Update nf_core/components/list.py Co-authored-by: awgymer <24782660+awgymer@users.noreply.github.com> --- nf_core/components/list.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index 401eade78a..eeacb9c39b 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -18,7 +18,8 @@ def __init__(self, component_type, pipeline_dir, remote=True, remote_url=None, b super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) self.remote = remote - def list_components(self, keywords=[], print_json=False): + def list_components(self, keywords=None, print_json=False): + keywords = keywords or [] """ Get available modules/subworkflows names from GitHub tree for repo and print as list to stdout From 665402779a432d9c1fce2580f0826578a5079cbd Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 3 Nov 2022 19:08:24 +0100 Subject: [PATCH 446/854] Apply suggestions from code review --- .github/CONTRIBUTING.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 19ccba5566..0b1d45f874 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -128,18 +128,17 @@ nf-core create -n testpipeline -d "This pipeline is for testing" nf-core lint nf-core-testpipeline ``` -# To get started: -- Open the repo in [Codespaces](https://github.com/{{ name }})/codespaces) -- Tools installed - - nf-core - - nextflow +## GitHub Codespaces -**Note:** +This repo includes a devcontainer configuration which will create a GitHub Codespaces for Nextflow development! This is an online developer environment that runs in your browser, complete with VSCode and a terminal. -## Codespaces +To get started: -This repo includes a devcontainer configuration which will create a GitHub Codespace for Nextflow development! +- Open the repo in [Codespaces](https://github.com/nf-core/tools/codespaces) +- Tools installed + - nf-core + - nextflow Devcontainer specs: From 0146a89772aed6aaf64f093ba6e207aa4eb5d909 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 3 Nov 2022 19:10:14 +0100 Subject: [PATCH 447/854] Add Codespaces instructions to pipeline template CONTRIBUTING too --- .github/CONTRIBUTING.md | 3 +-- .../pipeline-template/.github/CONTRIBUTING.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 0b1d45f874..fc73294b5a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -128,7 +128,6 @@ nf-core create -n testpipeline -d "This pipeline is for testing" nf-core lint nf-core-testpipeline ``` - ## GitHub Codespaces This repo includes a devcontainer configuration which will create a GitHub Codespaces for Nextflow development! This is an online developer environment that runs in your browser, complete with VSCode and a terminal. @@ -138,7 +137,7 @@ To get started: - Open the repo in [Codespaces](https://github.com/nf-core/tools/codespaces) - Tools installed - nf-core - - nextflow + - Nextflow Devcontainer specs: diff --git a/nf_core/pipeline-template/.github/CONTRIBUTING.md b/nf_core/pipeline-template/.github/CONTRIBUTING.md index b9720ac70b..9afdd2987b 100644 --- a/nf_core/pipeline-template/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/.github/CONTRIBUTING.md @@ -109,3 +109,19 @@ If you are using a new feature from core Nextflow, you may bump the minimum requ ### Images and figures For overview images and other documents we follow the nf-core [style guidelines and examples](https://nf-co.re/developers/design_guidelines). + +## GitHub Codespaces + +This repo includes a devcontainer configuration which will create a GitHub Codespaces for Nextflow development! This is an online developer environment that runs in your browser, complete with VSCode and a terminal. + +To get started: + +- Open the repo in [Codespaces](https://github.com/{{ name }}/codespaces) +- Tools installed + - nf-core + - Nextflow + +Devcontainer specs: + +- [DevContainer config](.devcontainer/devcontainer.json) +- [Dockerfile](.devcontainer/Dockerfile) From d6e35d18d1bf23699ec594b2750d6b9917541d7b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 09:41:36 +0100 Subject: [PATCH 448/854] some linting modifications in imports --- nf_core/components/components_install.py | 7 +------ nf_core/modules/install.py | 3 --- nf_core/modules/update.py | 2 +- nf_core/subworkflows/install.py | 3 --- tests/modules/update.py | 2 +- tests/test_subworkflows.py | 3 +-- 6 files changed, 4 insertions(+), 16 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 9cd40005d2..9e4486c79a 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -1,12 +1,7 @@ -import glob -import json import logging import os -import re -import jinja2 import questionary -import rich import nf_core.modules.modules_utils import nf_core.utils @@ -76,7 +71,7 @@ def get_version(component, component_type, sha, prompt, current_version, modules version = sha elif prompt: try: - version = nf_core.modules.modules_utils.prompt_component_version_sha( + version = prompt_component_version_sha( component, component_type, installed_sha=current_version, diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index e928eb4c79..2ce5e596fa 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -1,15 +1,12 @@ import logging import os -import questionary - import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.modules_json import ModulesJson from .modules_command import ModuleCommand -from .modules_repo import NF_CORE_MODULES_NAME log = logging.getLogger(__name__) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 8fb3b45d49..978097c0a0 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -152,7 +152,7 @@ def update(self, module=None): if sha is not None: version = sha elif self.prompt: - version = nf_core.modules.modules_utils.prompt_component_version_sha( + version = prompt_component_version_sha( module, "modules", modules_repo=modules_repo, installed_sha=current_version ) else: diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 439cf187b7..6fbd2370b6 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -3,14 +3,11 @@ import re from pathlib import Path -import questionary - import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME from .subworkflows_command import SubworkflowCommand diff --git a/tests/modules/update.py b/tests/modules/update.py index 9ff098d88e..89640fbd24 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -214,7 +214,7 @@ def test_update_with_config_no_updates(self): def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" - install_obj = nf_core.modules.ModuleInstall( + install_obj = ModuleInstall( self.pipeline_dir, prompt=False, force=False, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index ea4fa986dd..37d57e4f4b 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -3,7 +3,6 @@ """ import os -import shutil import tempfile import unittest @@ -13,7 +12,7 @@ import nf_core.modules import nf_core.subworkflows -from .utils import GITLAB_URL, mock_api_calls +from .utils import mock_api_calls def create_modules_repo_dummy(tmp_dir): From 19d0ae1ce15b01ddce45b88f93fea962da588913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 4 Nov 2022 09:49:47 +0100 Subject: [PATCH 449/854] Update tests/test_subworkflows.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/test_subworkflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index fbe33dbe0f..e86ba82457 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -14,7 +14,7 @@ from .utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, mock_api_calls -def create_subworkflows_repo_dummy(tmp_dir): +def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" root_dir = os.path.join(tmp_dir, "modules") From 4be837dde411592aa4a5c6967508255e0e45ab78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 4 Nov 2022 09:51:42 +0100 Subject: [PATCH 450/854] Update tests/test_subworkflows.py --- tests/test_subworkflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index e86ba82457..87e3e5ca28 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -52,7 +52,7 @@ def setUp(self): ).init_pipeline() # Set up the nf-core/modules repo dummy - self.nfcore_modules = create_subworkflows_repo_dummy(self.tmp_dir) + self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) # Set up install objects self.sw_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=False) From 69f02169e4969a0adb1e0c76e26c7bf579940b89 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 4 Nov 2022 10:14:49 +0100 Subject: [PATCH 451/854] update gitlab subworkflow names --- tests/subworkflows/list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index bbb2f82235..8daf5fb599 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -29,7 +29,7 @@ def test_subworkflows_list_remote_gitlab(self): def test_subworkflows_install_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - self.sw_install.install("bam_stats_samtools") + self.sw_install.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() console = Console(record=True) @@ -40,7 +40,7 @@ def test_subworkflows_install_and_list_subworkflows(self): def test_subworkflows_install_gitlab_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - self.sw_install_gitlab.install("bam_stats_samtools") + self.sw_install_gitlab.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() console = Console(record=True) From d426153aaa4b51df8f32797098e5966a416452fe Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 4 Nov 2022 10:31:03 +0100 Subject: [PATCH 452/854] fix deprecation warnings for python and github actions --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/sync.yml | 4 ++-- nf_core/create.py | 4 ++-- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 4 ++-- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- .../pipeline-template/.github/workflows/linting_comment.yml | 2 +- requirements.txt | 1 + 8 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 97adc7ac71..d6a835ad0b 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -121,7 +121,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: nf-core-log-file path: log.txt diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 603403b505..b4b12621ce 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -51,7 +51,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: nf-core-log-file path: log.txt diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index c95ef5e910..94bc133f4c 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -27,7 +27,7 @@ jobs: else curl -O https://nf-co.re/pipeline_names.json fi - echo "::set-output name=matrix::$(cat pipeline_names.json)" + echo "name=matrix::$(cat pipeline_names.json)" >> $GITHUB_OUTPUT sync: needs: get-pipelines @@ -78,7 +78,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/nf_core/create.py b/nf_core/create.py index feb0e25cbb..ca06bd9c36 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -3,7 +3,6 @@ organization's specification based on a template. """ import configparser -import imghdr import logging import os import random @@ -13,6 +12,7 @@ import time from pathlib import Path +import filetype import git import jinja2 import questionary @@ -492,7 +492,7 @@ def download_pipeline_logo(self, url, img_fn): with open(img_fn, "wb") as fh: fh.write(r.content) # Check that the file looks valid - image_type = imghdr.what(img_fn) + image_type = filetype.guess(img_fn).extension if image_type != "png": log.error(f"Logo from the website didn't look like an image: '{image_type}'") continue diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 5ad82fe8f8..f136e67e80 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "::set-output name=result::pass" + echo "name=result::pass" >> $GITHUB_OUTPUT else - echo "::set-output name=result::fail" + echo "name=result::fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 8f9a4173bd..766fd36b8a 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -99,7 +99,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: linting-logs path: | diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 5b91eedce0..585b933f1c 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "::set-output name=pr_number::$(cat linting-logs/PR_number.txt)" + run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 diff --git a/requirements.txt b/requirements.txt index 0a4a5fb7e7..c60791b0e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ click +filetype galaxy-tool-util GitPython jinja2 From 5f8108f84d16cf002b353cfd38a0abca27e47848 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 10:48:17 +0100 Subject: [PATCH 453/854] add installed_by tracking --- nf_core/modules/install.py | 7 ++++++- nf_core/modules/modules_json.py | 12 ++++++++++-- nf_core/modules/update.py | 4 ++-- nf_core/subworkflows/install.py | 11 ++++++++++- tests/modules/modules_json.py | 5 +++-- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 2ce5e596fa..80cb426677 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -21,11 +21,16 @@ def __init__( remote_url=None, branch=None, no_pull=False, + installed_by=False, ): super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha + if installed_by: + self.installed_by = installed_by + else: + self.installed_by = self.component_type def install(self, module, silent=False): if self.repo_type == "modules": @@ -100,5 +105,5 @@ def install(self, module, silent=False): # Update module.json with newly installed module modules_json.load() - modules_json.update(self.modules_repo, module, version) + modules_json.update(self.modules_repo, module, version, self.installed_by) return True diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 96a62dd2ef..23e54249b0 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -518,7 +518,7 @@ def load(self): except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") - def update(self, modules_repo, module_name, module_version, write_file=True): + def update(self, modules_repo, module_name, module_version, installed_by, write_file=True): """ Updates the 'module.json' file with new module info @@ -540,13 +540,17 @@ def update(self, modules_repo, module_name, module_version, write_file=True): repo_modules_entry[module_name] = {} repo_modules_entry[module_name]["git_sha"] = module_version repo_modules_entry[module_name]["branch"] = branch + try: + repo_modules_entry[module_name]["installed"].add(installed_by) + except KeyError: + repo_modules_entry[module_name]["installed"] = set(installed_by) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) if write_file: self.dump() - def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version, write_file=True): + def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version, installed_by, write_file=True): """ Updates the 'module.json' file with new subworkflow info @@ -571,6 +575,10 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version repo_subworkflows_entry[subworkflow_name] = {} repo_subworkflows_entry[subworkflow_name]["git_sha"] = subworkflow_version repo_subworkflows_entry[subworkflow_name]["branch"] = branch + try: + repo_subworkflows_entry[subworkflow_name]["installed"].add(installed_by) + except KeyError: + repo_subworkflows_entry[subworkflow_name]["installed"] = set(installed_by) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 978097c0a0..8aea009efd 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -223,10 +223,10 @@ def update(self, module=None): # Clear the module directory and move the installed files there self.move_files_from_tmp_dir(module, install_tmp_dir, modules_repo.repo_path, version) # Update modules.json with newly installed module - self.modules_json.update(modules_repo, module, version) + self.modules_json.update(modules_repo, module, version, self.component_type) else: # Don't save to a file, just iteratively update the variable - self.modules_json.update(modules_repo, module, version, write_file=False) + self.modules_json.update(modules_repo, module, version, self.component_type, write_file=False) if self.save_diff_fn: # Write the modules.json diff to the file diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 6fbd2370b6..ba4a6bc6d0 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -24,11 +24,16 @@ def __init__( remote_url=None, branch=None, no_pull=False, + installed_by=False, ): super().__init__(pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha + if installed_by: + self.installed_by = installed_by + else: + self.installed_by = self.component_type def install(self, subworkflow, silent=False): if self.repo_type == "modules": @@ -106,7 +111,7 @@ def install(self, subworkflow, silent=False): # Update module.json with newly installed subworkflow modules_json.load() - modules_json.update_subworkflow(self.modules_repo, subworkflow, version) + modules_json.update_subworkflow(self.modules_repo, subworkflow, version, self.installed_by) return True def get_modules_subworkflows_to_install(self, subworkflow_dir): @@ -136,7 +141,10 @@ def install_included_components(self, subworkflow_dir): """ modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) for s_install in subworkflows_to_install: + original_installed = self.installed_by + self.installed_by = Path(subworkflow_dir).parts[-1] self.install(s_install, silent=True) + self.installed_by = original_installed for m_install in modules_to_install: module_install = ModuleInstall( self.dir, @@ -145,5 +153,6 @@ def install_included_components(self, subworkflow_dir): sha=self.sha, remote_url=self.modules_repo.remote_url, branch=self.modules_repo.branch, + installed_by=Path(subworkflow_dir).parts[-1], ) module_install.install(m_install, silent=True) diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 60fb3006c1..08ef3d9dc3 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -32,7 +32,7 @@ def test_mod_json_update(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the modules.json file mod_repo_obj = ModulesRepo() - mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", False) + mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", "modules", False) mod_json = mod_json_obj.get_modules_json() assert "MODULE_NAME" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"] @@ -41,6 +41,7 @@ def test_mod_json_update(self): NF_CORE_MODULES_DEFAULT_BRANCH == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["branch"] ) + assert "modules" == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["installed"] def test_mod_json_create(self): @@ -154,7 +155,7 @@ def test_mod_json_up_to_date_reinstall_fails(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the fastqc module entry to an invalid git_sha - mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", True) + mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", True) # Remove the fastqc module fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") From 363c1a895ae6de095a67980f9cf2015041e89722 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 4 Nov 2022 10:50:49 +0100 Subject: [PATCH 454/854] update isort and setup-node actions --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/fix-linting.yml | 4 ++-- .github/workflows/lint-code.yml | 6 +++--- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index d6a835ad0b..4dc87c0a14 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,7 +48,7 @@ jobs: version: ${{ matrix.NXF_VER }} # Install the Prettier linting tools - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install Prettier run: npm install -g prettier diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 674463f675..947555fc45 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -24,7 +24,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install Prettier run: npm install -g prettier @prettier/plugin-php @@ -43,7 +43,7 @@ jobs: with: python-version: 3.8 - name: python-isort - uses: isort/isort-action@v0.1.0 + uses: isort/isort-action@v1.0.0 with: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index a7cda448c4..2ca5286fd4 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -32,7 +32,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install Prettier run: npm install -g prettier @@ -81,7 +81,7 @@ jobs: with: python-version: 3.8 - name: python-isort - uses: isort/isort-action@v0.1.0 + uses: isort/isort-action@v1.0.0 with: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index f136e67e80..4c586a0929 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -24,7 +24,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install Prettier run: npm install -g prettier @prettier/plugin-php diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 766fd36b8a..7e0a495789 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -29,7 +29,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 - name: Install Prettier run: npm install -g prettier From 36df3fb73c222d686afc23ab68d01b020f0cc8f8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 10:54:30 +0100 Subject: [PATCH 455/854] fix some linting issues --- nf_core/subworkflows/create.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 7181498ef5..481bd1ea94 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -9,7 +9,6 @@ import os import yaml -from packaging.version import parse as parse_version import nf_core import nf_core.components.components_create @@ -114,7 +113,7 @@ def create(self): pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError as e: + except FileNotFoundError: raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") new_files = list(self.file_paths.values()) From ca63f067ef3dfbbd97e474e4e224c5d49075ab2e Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 4 Nov 2022 10:59:33 +0100 Subject: [PATCH 456/854] upgrade setup-python --- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 947555fc45..30cf965af0 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -39,7 +39,7 @@ jobs: options: "--color" - name: Set up Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: python-isort diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 2ca5286fd4..89aa48462b 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -77,7 +77,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: python-isort From 535b0bc6ef2620364af75d294d903b4162a647b7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 11:38:30 +0100 Subject: [PATCH 457/854] use list instead of set --- nf_core/modules/modules_json.py | 10 ++++++---- tests/modules/modules_json.py | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 23e54249b0..16312a881c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -541,9 +541,10 @@ def update(self, modules_repo, module_name, module_version, installed_by, write_ repo_modules_entry[module_name]["git_sha"] = module_version repo_modules_entry[module_name]["branch"] = branch try: - repo_modules_entry[module_name]["installed"].add(installed_by) + if installed_by not in repo_modules_entry[module_name]["installed"]: + repo_modules_entry[module_name]["installed"].append(installed_by) except KeyError: - repo_modules_entry[module_name]["installed"] = set(installed_by) + repo_modules_entry[module_name]["installed"] = [installed_by] # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) @@ -576,9 +577,10 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version repo_subworkflows_entry[subworkflow_name]["git_sha"] = subworkflow_version repo_subworkflows_entry[subworkflow_name]["branch"] = branch try: - repo_subworkflows_entry[subworkflow_name]["installed"].add(installed_by) + if installed_by not in repo_subworkflows_entry[subworkflow_name]["installed"]: + repo_subworkflows_entry[subworkflow_name]["installed"].append(installed_by) except KeyError: - repo_subworkflows_entry[subworkflow_name]["installed"] = set(installed_by) + repo_subworkflows_entry[subworkflow_name]["installed"] = [installed_by] # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 08ef3d9dc3..a42d0258bd 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -41,7 +41,8 @@ def test_mod_json_update(self): NF_CORE_MODULES_DEFAULT_BRANCH == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["branch"] ) - assert "modules" == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["installed"] + print(mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]) + assert "modules" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["installed"] def test_mod_json_create(self): From 82c5f5cf2906f011dc8707b61990852afee0815c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 12:46:56 +0100 Subject: [PATCH 458/854] add source even if the module/subworkflow is already installed --- nf_core/components/components_install.py | 2 +- nf_core/modules/install.py | 5 +++++ nf_core/subworkflows/install.py | 13 ++++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index a239a97ed7..d4e965fd3c 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -100,4 +100,4 @@ def clean_modules_json(component, component_type, modules_repo, modules_json): f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" ) modules_json.remove_entry(component, repo_to_remove, modules_repo.repo_path) - break + return diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 5eed2866e4..cccae4fbc2 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -73,6 +73,11 @@ def install(self, module, silent=False): if not nf_core.components.components_install.check_component_installed( self.component_type, module, current_version, module_dir, self.modules_repo, self.force, self.prompt ): + log.debug( + f"Module is already installed and force is not set.\nAdding the new installation source {self.installed_by} for module {module} to 'modules.json' without installing the module." + ) + modules_json.load() + modules_json.update(self.modules_repo, module, current_version, self.installed_by) return False version = nf_core.components.components_install.get_version( diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index d272b33741..a7139b0097 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -79,6 +79,11 @@ def install(self, subworkflow, silent=False): self.force, self.prompt, ): + log.debug( + f"Subworkflow is already installed and force is not set.\nAdding the new installation source {self.installed_by} for subworkflow {subworkflow} to 'modules.json' without installing the subworkflow." + ) + modules_json.load() + modules_json.update(self.modules_repo, subworkflow, current_version, self.installed_by) return False version = nf_core.components.components_install.get_version( @@ -89,7 +94,13 @@ def install(self, subworkflow, silent=False): # Remove subworkflow if force is set and component is installed if self.force and nf_core.components.components_install.check_component_installed( - self.component_type, subworkflow, current_version, subworkflow_dir, self.modules_repo, self.force + self.component_type, + subworkflow, + current_version, + subworkflow_dir, + self.modules_repo, + self.force, + self.prompt, ): log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") self.clear_component_dir(subworkflow, subworkflow_dir) From ee9ae31cce6fb264b64b1978b96a054d001016ae Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 13:11:48 +0100 Subject: [PATCH 459/854] fix bug with always 'recomputing SHA' --- nf_core/modules/modules_json.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 0c63d386ce..25c8dd3c40 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -373,25 +373,32 @@ def parse_dirs(self, dirs, missing_installation, component_type): component_in_file = False git_url = None for repo in missing_installation: - for dir_name in missing_installation[repo][component_type]: - if component in missing_installation[repo][component_type][dir_name]: - component_in_file = True - git_url = repo - break + if component_type in missing_installation[repo]: + for dir_name in missing_installation[repo][component_type]: + if component in missing_installation[repo][component_type][dir_name]: + component_in_file = True + git_url = repo + break if not component_in_file: - # If it is not, add it to the list of missing subworkflow + # If it is not, add it to the list of missing components untracked_dirs.append(component) else: - # If it does, remove the subworkflow from missing_installation + # If it does, remove the component from missing_installation module_repo = missing_installation[git_url] # Check if the entry has a git sha and branch before removing components_dict = module_repo[component_type][install_dir] if "git_sha" not in components_dict[component] or "branch" not in components_dict[component]: self.determine_module_branches_and_shas(component, git_url, module_repo["base_path"], [component]) - # Remove the subworkflow from subworkflows without installation + # Remove the component from components without installation module_repo[component_type][install_dir].pop(component) if len(module_repo[component_type][install_dir]) == 0: + # If no modules/subworkflows with missing installation left, remove the install_dir from missing_installation + missing_installation[git_url][component_type].pop(install_dir) + if len(module_repo[component_type]) == 0: + # If no modules/subworkflows with missing installation left, remove the component_type from missing_installation + missing_installation[git_url].pop(component_type) + if len(module_repo) == 0: # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation missing_installation.pop(git_url) From e40b80267235725d5d3bd7c48b08fc6ea44078f2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 13:12:47 +0100 Subject: [PATCH 460/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e3448581..4e534dfab1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Run tests with Python 3.11 ([#1970](https://github.com/nf-core/tools/pull/1970)) - Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971)) - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out` +- Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999)) ### Modules From 0cb21e1a88f134552f9b6bc64c070a16937c6d56 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 13:13:39 +0100 Subject: [PATCH 461/854] modify pipeline template --- nf_core/pipeline-template/modules.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 4037d5905f..6a1bf1f96a 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,15 +7,18 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "installed": ["modules"] } } } From 53057702b20c1cbd543e56cf5f3523b58468e756 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 13:24:41 +0100 Subject: [PATCH 462/854] don't remove entry from modules.json with force to avoid lose installed track --- nf_core/components/components_install.py | 2 +- nf_core/modules/install.py | 6 +++--- nf_core/modules/modules_json.py | 17 ++++++++++------- nf_core/modules/remove.py | 4 ++-- nf_core/subworkflows/install.py | 6 +++--- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index d4e965fd3c..bfecb73e77 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -99,5 +99,5 @@ def clean_modules_json(component, component_type, modules_repo, modules_json): log.info( f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" ) - modules_json.remove_entry(component, repo_to_remove, modules_repo.repo_path) + modules_json.remove_entry(component_type, component, repo_to_remove, modules_repo.repo_path) return diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index cccae4fbc2..b7d2e3bde0 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -90,15 +90,15 @@ def install(self, module, silent=False): if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") self.clear_component_dir(module, module_dir) - nf_core.components.components_install.clean_modules_json( - module, self.component_type, self.modules_repo, modules_json - ) log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'") log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}") # Download module files if not self.install_component_files(module, version, self.modules_repo, install_folder): + nf_core.components.components_install.clean_modules_json( + module, self.component_type, self.modules_repo, modules_json + ) return False if not silent: diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 25c8dd3c40..8ccb81f5b4 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -597,12 +597,13 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version if write_file: self.dump() - def remove_entry(self, module_name, repo_url, install_dir): + def remove_entry(self, component_type, name, repo_url, install_dir): """ Removes an entry from the 'modules.json' file. Args: - module_name (str): Name of the module to be removed + component_type (Str): Type of component [modules, subworkflows] + name (str): Name of the module to be removed repo_url (str): URL of the repository containing the module install_dir (str): Name of the directory where modules are installed Returns: @@ -612,15 +613,17 @@ def remove_entry(self, module_name, repo_url, install_dir): return False if repo_url in self.modules_json.get("repos", {}): repo_entry = self.modules_json["repos"][repo_url] - if module_name in repo_entry["modules"].get(install_dir, {}): - repo_entry["modules"][install_dir].pop(module_name) + if name in repo_entry[component_type].get(install_dir, {}): + repo_entry[component_type][install_dir].pop(name) else: - log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.") + log.warning( + f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file." + ) return False - if len(repo_entry["modules"][install_dir]) == 0: + if len(repo_entry[component_type][install_dir]) == 0: self.modules_json["repos"].pop(repo_url) else: - log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.") + log.warning(f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file.") return False self.dump() diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 1284c6d59c..8afe634257 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -50,13 +50,13 @@ def remove(self, module): if modules_json.module_present(module, self.modules_repo.remote_url, repo_path): log.error(f"Found entry for '{module}' in 'modules.json'. Removing...") - modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) + modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path) return False log.info(f"Removing {module}") # Remove entry from modules.json - modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path) + modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path) # Remove the module return self.clear_component_dir(module, module_dir) diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index a7139b0097..d58327c877 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -104,15 +104,15 @@ def install(self, subworkflow, silent=False): ): log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") self.clear_component_dir(subworkflow, subworkflow_dir) - nf_core.components.components_install.clean_modules_json( - subworkflow, self.component_type, self.modules_repo, modules_json - ) log.info(f"{'Rei' if self.force else 'I'}nstalling '{subworkflow}'") log.debug(f"Installing subworkflow '{subworkflow}' at hash {version} from {self.modules_repo.remote_url}") # Download subworkflow files if not self.install_component_files(subworkflow, version, self.modules_repo, install_folder): + nf_core.components.components_install.clean_modules_json( + subworkflow, self.component_type, self.modules_repo, modules_json + ) return False # Install included modules and subworkflows From a8cc0a542bbdbfa65d26d6c4849920ef0a022d11 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 13:36:55 +0100 Subject: [PATCH 463/854] add 'installed' to modules.json for already installed components --- nf_core/modules/modules_json.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 8ccb81f5b4..d14f4bb2f2 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -476,6 +476,9 @@ def check_up_to_date(self): If a module/subworkflow is installed but the entry in 'modules.json' is missing we iterate through the commit log in the remote to try to determine the SHA. + + Check that we have the "installed" value in 'modules.json', otherwise add it. + Assume that the modules/subworkflows were installed by and nf-core command (don't track installed by subworkflows). """ try: self.load() @@ -510,6 +513,16 @@ def check_up_to_date(self): if len(subworkflows_missing_from_modules_json) > 0: self.resolve_missing_from_modules_json(subworkflows_missing_from_modules_json, "subworkflows") + # If the "installed" value is not present for modules/subworkflows, add it. + for repo, repo_content in self.modules_json["repos"].items(): + for component_type, dir_content in repo_content.items(): + for install_dir, installed_components in dir_content.items(): + for component, component_features in installed_components.items(): + if "installed" not in component_features: + self.modules_json["repos"][repo][component_type][install_dir][component]["installed"] = [ + component_type + ] + self.dump() def load(self): From 2eff7a49bfd6d50e22becb9ce2417ed131410e3c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 15:38:36 +0100 Subject: [PATCH 464/854] fix tests --- nf_core/components/components_create.py | 1 - nf_core/components/components_install.py | 4 ++-- nf_core/modules/install.py | 9 +++++---- nf_core/modules/modules_json.py | 24 +++++++++++++++++++++--- nf_core/subworkflows/install.py | 9 +++++---- tests/modules/list.py | 1 - tests/modules/modules_json.py | 5 ++--- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py index 1206c7c415..3452665f55 100644 --- a/nf_core/components/components_create.py +++ b/nf_core/components/components_create.py @@ -99,7 +99,6 @@ def get_component_dirs(component_type, repo_type, directory, name, supername, su # Check whether component file already exists component_file = os.path.join(local_component_dir, f"{name}.nf") if os.path.exists(component_file) and not force_overwrite: - print(f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite") raise UserWarning( f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" ) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index bfecb73e77..7680134d14 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -93,11 +93,11 @@ def clean_modules_json(component, component_type, modules_repo, modules_json): """ for repo_url, repo_content in modules_json.modules_json["repos"].items(): for dir, dir_components in repo_content[component_type].items(): - for name, _ in dir_components.items(): + for name, component_values in dir_components.items(): if name == component and dir == modules_repo.repo_path: repo_to_remove = repo_url log.info( f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" ) modules_json.remove_entry(component_type, component, repo_to_remove, modules_repo.repo_path) - return + return component_values["installed"] diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index b7d2e3bde0..e0919e07dd 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -87,18 +87,19 @@ def install(self, module, silent=False): return False # Remove module if force is set + install_track = None if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") self.clear_component_dir(module, module_dir) + install_track = nf_core.components.components_install.clean_modules_json( + module, self.component_type, self.modules_repo, modules_json + ) log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'") log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}") # Download module files if not self.install_component_files(module, version, self.modules_repo, install_folder): - nf_core.components.components_install.clean_modules_json( - module, self.component_type, self.modules_repo, modules_json - ) return False if not silent: @@ -110,5 +111,5 @@ def install(self, module, silent=False): # Update module.json with newly installed module modules_json.load() - modules_json.update(self.modules_repo, module, version, self.installed_by) + modules_json.update(self.modules_repo, module, version, self.installed_by, install_track) return True diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index d14f4bb2f2..eabaa51c73 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -274,7 +274,11 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): found_sha = True break if found_sha: - repo_entry[module] = {"branch": modules_repo.branch, "git_sha": correct_commit_sha} + repo_entry[module] = { + "branch": modules_repo.branch, + "git_sha": correct_commit_sha, + "installed": "modules", + } # Clean up the modules we were unable to find the sha for for module in sb_local: @@ -541,7 +545,7 @@ def load(self): except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") - def update(self, modules_repo, module_name, module_version, installed_by, write_file=True): + def update(self, modules_repo, module_name, module_version, installed_by, installed_by_log=None, write_file=True): """ Updates the 'module.json' file with new module info @@ -549,8 +553,12 @@ def update(self, modules_repo, module_name, module_version, installed_by, write_ modules_repo (ModulesRepo): A ModulesRepo object configured for the new module module_name (str): Name of new module module_version (str): git SHA for the new module entry + installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json' write_file (bool): whether to write the updated modules.json to a file. """ + if installed_by_log is None: + installed_by_log = [] + if self.modules_json is None: self.load() repo_name = modules_repo.repo_path @@ -568,13 +576,17 @@ def update(self, modules_repo, module_name, module_version, installed_by, write_ repo_modules_entry[module_name]["installed"].append(installed_by) except KeyError: repo_modules_entry[module_name]["installed"] = [installed_by] + finally: + repo_modules_entry[module_name]["installed"].extend(installed_by_log) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) if write_file: self.dump() - def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version, installed_by, write_file=True): + def update_subworkflow( + self, modules_repo, subworkflow_name, subworkflow_version, installed_by, installed_by_log=None, write_file=True + ): """ Updates the 'module.json' file with new subworkflow info @@ -582,8 +594,12 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version modules_repo (ModulesRepo): A ModulesRepo object configured for the new subworkflow subworkflow_name (str): Name of new subworkflow subworkflow_version (str): git SHA for the new subworkflow entry + installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json' write_file (bool): whether to write the updated modules.json to a file. """ + if installed_by_log is None: + installed_by_log = [] + if self.modules_json is None: self.load() repo_name = modules_repo.repo_path @@ -604,6 +620,8 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version repo_subworkflows_entry[subworkflow_name]["installed"].append(installed_by) except KeyError: repo_subworkflows_entry[subworkflow_name]["installed"] = [installed_by] + finally: + repo_subworkflows_entry[subworkflow_name]["installed"].extend(installed_by_log) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index d58327c877..750007f294 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -93,6 +93,7 @@ def install(self, subworkflow, silent=False): return False # Remove subworkflow if force is set and component is installed + install_track = None if self.force and nf_core.components.components_install.check_component_installed( self.component_type, subworkflow, @@ -104,15 +105,15 @@ def install(self, subworkflow, silent=False): ): log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") self.clear_component_dir(subworkflow, subworkflow_dir) + install_track = nf_core.components.components_install.clean_modules_json( + subworkflow, self.component_type, self.modules_repo, modules_json + ) log.info(f"{'Rei' if self.force else 'I'}nstalling '{subworkflow}'") log.debug(f"Installing subworkflow '{subworkflow}' at hash {version} from {self.modules_repo.remote_url}") # Download subworkflow files if not self.install_component_files(subworkflow, version, self.modules_repo, install_folder): - nf_core.components.components_install.clean_modules_json( - subworkflow, self.component_type, self.modules_repo, modules_json - ) return False # Install included modules and subworkflows @@ -130,7 +131,7 @@ def install(self, subworkflow, silent=False): # Update module.json with newly installed subworkflow modules_json.load() - modules_json.update_subworkflow(self.modules_repo, subworkflow, version, self.installed_by) + modules_json.update_subworkflow(self.modules_repo, subworkflow, version, self.installed_by, install_track) return True def get_modules_subworkflows_to_install(self, subworkflow_dir): diff --git a/tests/modules/list.py b/tests/modules/list.py index 2cd1333faf..d92cd58dd5 100644 --- a/tests/modules/list.py +++ b/tests/modules/list.py @@ -19,7 +19,6 @@ def test_modules_list_remote_gitlab(self): """Test listing the modules in the remote gitlab repo""" mods_list = nf_core.modules.ModuleList(None, remote=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH) listed_mods = mods_list.list_components() - print(f"listed modules are {listed_mods}") console = Console(record=True) console.print(listed_mods) output = console.export_text() diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index a42d0258bd..3b6be3101f 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -32,7 +32,7 @@ def test_mod_json_update(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the modules.json file mod_repo_obj = ModulesRepo() - mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", "modules", False) + mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", "modules", write_file=False) mod_json = mod_json_obj.get_modules_json() assert "MODULE_NAME" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"] @@ -41,7 +41,6 @@ def test_mod_json_update(self): NF_CORE_MODULES_DEFAULT_BRANCH == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["branch"] ) - print(mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]) assert "modules" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["installed"] @@ -156,7 +155,7 @@ def test_mod_json_up_to_date_reinstall_fails(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the fastqc module entry to an invalid git_sha - mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", True) + mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", write_file=True) # Remove the fastqc module fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") From b8f418ffa7af5e179a8e0e3cf6dd5321cb903ac5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 15:39:58 +0100 Subject: [PATCH 465/854] remove unused lib --- tests/modules/patch.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 0f7a7c9101..57af1c499e 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -1,4 +1,3 @@ -import json import os import tempfile from pathlib import Path @@ -234,15 +233,11 @@ def test_create_patch_update_success(self): "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) - with open(os.path.join(module_path, "main.nf"), "r") as fh: - print(fh.readlines()) # Update the module update_obj = nf_core.modules.ModuleUpdate( self.pipeline_dir, sha=SUCCEED_SHA, show_diff=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH ) assert update_obj.update(BISMARK_ALIGN) - with open(os.path.join(module_path, "main.nf"), "r") as fh: - print(fh.readlines()) # Check that a patch file with the correct name has been created assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} @@ -257,7 +252,6 @@ def test_create_patch_update_success(self): with open(module_path / patch_fn, "r") as fh: patch_lines = fh.readlines() module_relpath = module_path.relative_to(self.pipeline_dir) - print(patch_lines) assert f"--- {module_relpath / 'main.nf'}\n" in patch_lines assert f"+++ {module_relpath / 'main.nf'}\n" in patch_lines assert "- tuple val(meta), path(reads)\n" in patch_lines From 1b2d7f405e9deab6d1a48f0a233e7b3f2f1982f9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 15:40:48 +0100 Subject: [PATCH 466/854] remove prints --- docs/api/make_lint_md.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/api/make_lint_md.py b/docs/api/make_lint_md.py index e0265c707d..9b5a706473 100644 --- a/docs/api/make_lint_md.py +++ b/docs/api/make_lint_md.py @@ -21,7 +21,6 @@ def make_docs(docs_basedir, lint_tests, md_template): else: with open(fn, "w") as fh: fh.write(md_template.format(test_name)) - print(test_name) for fn in existing_docs: os.remove(fn) From 3d6ded489d6c03322ba62dca3451686162159a62 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:08:49 +0100 Subject: [PATCH 467/854] move pathc path function to components command and remove check_components_structure --- nf_core/components/components_command.py | 47 +++++++++++--- nf_core/modules/modules_command.py | 66 -------------------- nf_core/subworkflows/subworkflows_command.py | 3 - 3 files changed, 39 insertions(+), 77 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index d89fb4652e..d671131c16 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -158,22 +158,22 @@ def load_lint_config(self): except FileNotFoundError: log.debug(f"No lint config file found: {config_fn}") - def check_component_structure(self, component_name): + def check_modules_structure(self): """ - Check that the structure of the modules/subworkflow directory in a pipeline is the correct one: - modules/nf-core/TOOL/SUBTOOL | subworkflows/nf-core/SUBWORKFLOW + Check that the structure of the modules directory in a pipeline is the correct one: + modules/nf-core/TOOL/SUBTOOL Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: modules/nf-core/modules/TOOL/SUBTOOL """ if self.repo_type == "pipeline": wrong_location_modules = [] - for directory, _, files in os.walk(Path(self.dir, component_name)): + for directory, _, files in os.walk(Path(self.dir, "modules")): if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir, component_name)) + module_path = Path(directory).relative_to(Path(self.dir, "modules")) parts = module_path.parts # Check that there are modules installed directly under the 'modules' directory - if parts[1] == component_name: + if parts[1] == "modules": wrong_location_modules.append(module_path) # If there are modules installed in the wrong location if len(wrong_location_modules) > 0: @@ -185,12 +185,43 @@ def check_component_structure(self, component_name): ) # Move wrong modules to the right directory for module in wrong_location_modules: - modules_dir = Path(component_name).resolve() + modules_dir = Path("modules").resolve() correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) wrong_dir = Path(modules_dir, module) shutil.move(wrong_dir, correct_dir) log.info(f"Moved {wrong_dir} to {correct_dir}.") - shutil.rmtree(Path(self.dir, component_name, self.modules_repo.repo_path, component_name)) + shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() + + def check_patch_paths(self, patch_path, module_name): + """ + Check that paths in patch files are updated to the new modules path + """ + if patch_path.exists(): + log.info(f"Modules {module_name} contains a patch file.") + rewrite = False + with open(patch_path, "r") as fh: + lines = fh.readlines() + for index, line in enumerate(lines): + # Check if there are old paths in the patch file and replace + if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line: + rewrite = True + lines[index] = line.replace( + f"modules/{self.modules_repo.repo_path}/modules/{module_name}/", + f"modules/{self.modules_repo.repo_path}/{module_name}/", + ) + if rewrite: + log.info(f"Updating paths in {patch_path}") + with open(patch_path, "w") as fh: + for line in lines: + fh.write(line) + # Update path in modules.json if the file is in the correct format + modules_json = ModulesJson(self.dir) + modules_json.load() + if modules_json.has_git_url_and_modules(): + modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ + self.modules_repo.repo_path + ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) + modules_json.dump() diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index ef075a20b1..566af4f694 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -17,70 +17,4 @@ class ModuleCommand(ComponentCommand): def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): super().__init__("modules", dir, remote_url, branch, no_pull, hide_progress) - def check_patch_paths(self, patch_path, module_name): - """ - Check that paths in patch files are updated to the new modules path - """ - if patch_path.exists(): - log.info(f"Modules {module_name} contains a patch file.") - rewrite = False - with open(patch_path, "r") as fh: - lines = fh.readlines() - for index, line in enumerate(lines): - # Check if there are old paths in the patch file and replace - if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line: - rewrite = True - lines[index] = line.replace( - f"modules/{self.modules_repo.repo_path}/modules/{module_name}/", - f"modules/{self.modules_repo.repo_path}/{module_name}/", - ) - if rewrite: - log.info(f"Updating paths in {patch_path}") - with open(patch_path, "w") as fh: - for line in lines: - fh.write(line) - # Update path in modules.json if the file is in the correct format - modules_json = ModulesJson(self.dir) - modules_json.load() - if modules_json.has_git_url_and_modules(): - modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ - self.modules_repo.repo_path - ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) - modules_json.dump() - def check_modules_structure(self): - """ - Check that the structure of the modules directory in a pipeline is the correct one: - modules/nf-core/TOOL/SUBTOOL - - Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: - modules/nf-core/modules/TOOL/SUBTOOL - """ - if self.repo_type == "pipeline": - wrong_location_modules = [] - for directory, _, files in os.walk(Path(self.dir, "modules")): - if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir, "modules")) - parts = module_path.parts - # Check that there are modules installed directly under the 'modules' directory - if parts[1] == "modules": - wrong_location_modules.append(module_path) - # If there are modules installed in the wrong location - if len(wrong_location_modules) > 0: - log.info("The modules folder structure is outdated. Reinstalling modules.") - # Remove the local copy of the modules repository - log.info(f"Updating '{self.modules_repo.local_repo_dir}'") - self.modules_repo.setup_local_repo( - self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress - ) - # Move wrong modules to the right directory - for module in wrong_location_modules: - modules_dir = Path("modules").resolve() - correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) - wrong_dir = Path(modules_dir, module) - shutil.move(wrong_dir, correct_dir) - log.info(f"Moved {wrong_dir} to {correct_dir}.") - shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) - # Regenerate modules.json file - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py index e6d48d2a3f..2203707c09 100644 --- a/nf_core/subworkflows/subworkflows_command.py +++ b/nf_core/subworkflows/subworkflows_command.py @@ -1,7 +1,4 @@ import logging -import os -import shutil -from pathlib import Path from nf_core.components.components_command import ComponentCommand From 30021cb0eba480336dc3eb5bac39fd6128eaac49 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:20:51 +0100 Subject: [PATCH 468/854] initialise componentcommand with component_type --- nf_core/components/components_test.py | 2 +- nf_core/modules/bump_versions.py | 9 ++++----- nf_core/modules/create.py | 7 +++---- nf_core/modules/info.py | 4 ++-- nf_core/modules/install.py | 6 +++--- nf_core/modules/lint/__init__.py | 6 +++--- nf_core/modules/modules_command.py | 20 -------------------- nf_core/modules/modules_test.py | 6 +++--- nf_core/modules/patch.py | 6 +++--- nf_core/modules/remove.py | 7 +++++-- nf_core/modules/test_yml_builder.py | 6 +++--- nf_core/modules/update.py | 6 +++--- nf_core/subworkflows/create.py | 7 +++---- nf_core/subworkflows/install.py | 7 +++---- nf_core/subworkflows/subworkflows_command.py | 14 -------------- nf_core/subworkflows/subworkflows_test.py | 7 +++---- nf_core/subworkflows/test_yml_builder.py | 6 +++--- tests/modules/patch.py | 6 +++--- 18 files changed, 48 insertions(+), 84 deletions(-) delete mode 100644 nf_core/modules/modules_command.py delete mode 100644 nf_core/subworkflows/subworkflows_command.py diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index e8a600f785..a1f3481305 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -38,7 +38,7 @@ def __init__( self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__(".", remote_url, branch, no_pull) + super().__init__("component_type", ".", remote_url, branch, no_pull) def run(self): """Run test steps""" diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 5d064cf2ef..552b9106c8 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -18,17 +18,16 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.utils import plural_s as _s from nf_core.utils import rich_force_colors -from .modules_command import ModuleCommand - log = logging.getLogger(__name__) -class ModuleVersionBumper(ModuleCommand): - def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): - super().__init__(pipeline_dir, remote_url, branch, no_pull) +class ModuleVersionBumper(ComponentCommand): + def __init__(self, component_type, pipeline_dir, remote_url=None, branch=None, no_pull=False): + super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) self.up_to_date = None self.updated = None diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 1b1eeba5ea..24e2aaeed0 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -17,13 +17,12 @@ import nf_core.components.components_create import nf_core.modules.modules_utils import nf_core.utils - -from .modules_command import ModuleCommand +from nf_core.components.components_command import ComponentCommand log = logging.getLogger(__name__) -class ModuleCreate(ModuleCommand): +class ModuleCreate(ComponentCommand): def __init__( self, directory=".", @@ -36,7 +35,7 @@ def __init__( conda_version=None, repo_type=None, ): - super().__init__(directory) + super().__init__("modules", directory) self.directory = directory self.tool = tool self.author = author diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 8fe5c75ead..dec64fa772 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -13,14 +13,14 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson -from .modules_command import ModuleCommand +from nf_core.components.components_command import ComponentCommand from .modules_repo import NF_CORE_MODULES_REMOTE from .modules_utils import get_repo_type log = logging.getLogger(__name__) -class ModuleInfo(ModuleCommand): +class ModuleInfo(ComponentCommand): """ Class to print information of a module. diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index e0919e07dd..65cd9773bc 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -6,12 +6,12 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson -from .modules_command import ModuleCommand +from nf_core.components.components_command import ComponentCommand log = logging.getLogger(__name__) -class ModuleInstall(ModuleCommand): +class ModuleInstall(ComponentCommand): def __init__( self, pipeline_dir, @@ -23,7 +23,7 @@ def __init__( no_pull=False, installed_by=False, ): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index f4ab5583ca..d48f21ddca 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -22,7 +22,7 @@ import nf_core.modules.modules_utils import nf_core.utils from nf_core.lint_utils import console -from nf_core.modules.modules_command import ModuleCommand +from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo from nf_core.modules.nfcore_module import NFCoreModule @@ -48,7 +48,7 @@ def __init__(self, mod, lint_test, message, file_path): self.module_name = mod.module_name -class ModuleLint(ModuleCommand): +class ModuleLint(ComponentCommand): """ An object for linting modules either in a clone of the 'nf-core/modules' repository or in any nf-core pipeline directory @@ -73,7 +73,7 @@ def __init__( no_pull=False, hide_progress=False, ): - super().__init__(dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=False) + super().__init__("modules", dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=False) self.fail_warned = fail_warned self.passed = [] diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py deleted file mode 100644 index 566af4f694..0000000000 --- a/nf_core/modules/modules_command.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging -import os -import shutil -from pathlib import Path - -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson - -log = logging.getLogger(__name__) - - -class ModuleCommand(ComponentCommand): - """ - Base class for the 'nf-core modules' commands - """ - - def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): - super().__init__("modules", dir, remote_url, branch, no_pull, hide_progress) - - diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index 92227b992b..1bcb4f5b74 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -15,13 +15,13 @@ import nf_core.modules.modules_utils import nf_core.utils -from nf_core.modules.modules_command import ModuleCommand +from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) -class ModulesTest(ModuleCommand): +class ModulesTest(ComponentCommand): """ Class to run module pytests. @@ -61,7 +61,7 @@ def __init__( self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__(".", remote_url, branch, no_pull) + super().__init__("modules", ".", remote_url, branch, no_pull) def run(self): """Run test steps""" diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 84e839a033..db2243dc4f 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -7,17 +7,17 @@ import questionary import nf_core.utils +from nf_core.components.components_command import ComponentCommand -from .modules_command import ModuleCommand from .modules_differ import ModulesDiffer from .modules_json import ModulesJson log = logging.getLogger(__name__) -class ModulePatch(ModuleCommand): +class ModulePatch(ComponentCommand): def __init__(self, dir, remote_url=None, branch=None, no_pull=False): - super().__init__(dir, remote_url, branch, no_pull) + super().__init__("modules", dir, remote_url, branch, no_pull) self.modules_json = ModulesJson(dir) diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 8afe634257..afe16ca2d5 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -4,14 +4,17 @@ import questionary import nf_core.utils +from nf_core.components.components_command import ComponentCommand -from .modules_command import ModuleCommand from .modules_json import ModulesJson log = logging.getLogger(__name__) -class ModuleRemove(ModuleCommand): +class ModuleRemove(ComponentCommand): + def __init__(self): + super().__init__("modules") + def remove(self, module): """ Remove an already installed module diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index ce5bb9e7b0..13d415a2b0 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -24,14 +24,14 @@ from rich.syntax import Syntax import nf_core.utils +from nf_core.components.components_command import ComponentCommand -from .modules_command import ModuleCommand from .modules_repo import ModulesRepo log = logging.getLogger(__name__) -class ModulesTestYmlBuilder(ModuleCommand): +class ModulesTestYmlBuilder(ComponentCommand): def __init__( self, module_name=None, @@ -41,7 +41,7 @@ def __init__( force_overwrite=False, no_prompts=False, ): - super().__init__(directory) + super().__init__("modules", directory) self.module_name = module_name self.run_tests = run_tests self.test_yml_output_path = test_yml_output_path diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 223637e73b..297e893401 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -8,10 +8,10 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.utils import plural_es, plural_s, plural_y -from .modules_command import ModuleCommand from .modules_differ import ModulesDiffer from .modules_json import ModulesJson from .modules_repo import ModulesRepo @@ -19,7 +19,7 @@ log = logging.getLogger(__name__) -class ModuleUpdate(ModuleCommand): +class ModuleUpdate(ComponentCommand): def __init__( self, pipeline_dir, @@ -33,7 +33,7 @@ def __init__( branch=None, no_pull=False, ): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 481bd1ea94..2440b2ad8d 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -13,14 +13,13 @@ import nf_core import nf_core.components.components_create import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_utils import get_repo_type -from .subworkflows_command import SubworkflowCommand - log = logging.getLogger(__name__) -class SubworkflowCreate(SubworkflowCommand): +class SubworkflowCreate(ComponentCommand): def __init__( self, directory=".", @@ -29,7 +28,7 @@ def __init__( force=False, repo_type=None, ): - super().__init__(directory) + super().__init__("subworkflows", directory) self.directory = directory self.subworkflow = subworkflow self.author = author diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 750007f294..2df0c48970 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -6,15 +6,14 @@ import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson -from .subworkflows_command import SubworkflowCommand - log = logging.getLogger(__name__) -class SubworkflowInstall(SubworkflowCommand): +class SubworkflowInstall(ComponentCommand): def __init__( self, pipeline_dir, @@ -26,7 +25,7 @@ def __init__( no_pull=False, installed_by=False, ): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("subworkflows", pipeline_dir, remote_url, branch, no_pull) self.force = force self.prompt = prompt self.sha = sha diff --git a/nf_core/subworkflows/subworkflows_command.py b/nf_core/subworkflows/subworkflows_command.py deleted file mode 100644 index 2203707c09..0000000000 --- a/nf_core/subworkflows/subworkflows_command.py +++ /dev/null @@ -1,14 +0,0 @@ -import logging - -from nf_core.components.components_command import ComponentCommand - -log = logging.getLogger(__name__) - - -class SubworkflowCommand(ComponentCommand): - """ - Base class for the 'nf-core subworkflows' commands - """ - - def __init__(self, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): - super().__init__("subworkflows", dir, remote_url, branch, no_pull, hide_progress) diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 41ddd5bdcc..6ecf3aebf7 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -15,14 +15,13 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson -from .subworkflows_command import SubworkflowCommand - log = logging.getLogger(__name__) -class SubworkflowsTest(SubworkflowCommand): +class SubworkflowsTest(ComponentCommand): """ Class to run module pytests. @@ -62,7 +61,7 @@ def __init__( self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__(".", remote_url, branch, no_pull) + super().__init__("subworkflows", ".", remote_url, branch, no_pull) def run(self): """Run test steps""" diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 59729ba41d..e7752ab5a2 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -27,12 +27,12 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo -from nf_core.subworkflows.subworkflows_command import SubworkflowCommand +from nf_core.subworkflows.subworkflows_command import ComponentCommand log = logging.getLogger(__name__) -class SubworkflowTestYmlBuilder(SubworkflowCommand): +class SubworkflowTestYmlBuilder(ComponentCommand): def __init__( self, subworkflow=None, @@ -42,7 +42,7 @@ def __init__( force_overwrite=False, no_prompts=False, ): - super().__init__(directory) + super().__init__("subworkflows", directory) self.dir = directory self.subworkflow = subworkflow self.run_tests = run_tests diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 57af1c499e..4cd518c667 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -300,9 +300,9 @@ def test_create_patch_update_fail(self): # Check that the installed files have not been affected by the attempted patch temp_dir = Path(tempfile.mkdtemp()) - nf_core.modules.modules_command.ModuleCommand(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH).install_component_files( - BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir - ) + nf_core.modules.modules_command.ComponentCommand( + "modules", self.pipeline_dir, GITLAB_URL, PATCH_BRANCH + ).install_component_files(BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir) temp_module_dir = temp_dir / BISMARK_ALIGN for file in os.listdir(temp_module_dir): From c1643e7c37113a40a3541fd65349875b078a907f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:23:31 +0100 Subject: [PATCH 469/854] fix linting issues --- nf_core/components/components_test.py | 3 +++ nf_core/modules/bump_versions.py | 4 ++-- nf_core/modules/create.py | 2 +- nf_core/modules/info.py | 5 ++--- nf_core/modules/lint/__init__.py | 1 - 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index a1f3481305..7c6a9be649 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -1,3 +1,6 @@ +from nf_core.components.components_command import ComponentCommand + + class ComponentsTest(ComponentCommand): """ Class to run module pytests. diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 552b9106c8..aa09955f2d 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -26,8 +26,8 @@ class ModuleVersionBumper(ComponentCommand): - def __init__(self, component_type, pipeline_dir, remote_url=None, branch=None, no_pull=False): - super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.up_to_date = None self.updated = None diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 24e2aaeed0..a0e0fa66e4 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -156,7 +156,7 @@ def create(self): pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError as e: + except FileNotFoundError: raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") new_files = list(self.file_paths.values()) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index dec64fa772..2db3891ce0 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -11,9 +11,9 @@ from rich.text import Text import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson -from nf_core.components.components_command import ComponentCommand from .modules_repo import NF_CORE_MODULES_REMOTE from .modules_utils import get_repo_type @@ -56,7 +56,7 @@ class ModuleInfo(ComponentCommand): """ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): - super().__init__(pipeline_dir, remote_url, branch, no_pull) + super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) self.meta = None self.local_path = None self.remote_location = None @@ -139,7 +139,6 @@ def get_local_yaml(self): if self.repo_type == "pipeline": # Try to find and load the meta.yml file - repo_name = self.modules_repo.repo_path module_base_path = os.path.join(self.dir, "modules") # Check that we have any modules installed from this repo modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index d48f21ddca..271c09e66b 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -24,7 +24,6 @@ from nf_core.lint_utils import console from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import ModulesRepo from nf_core.modules.nfcore_module import NFCoreModule from nf_core.utils import plural_s as _s From 8eb8d30ec61f92225d69547874f49baedc127115 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:25:27 +0100 Subject: [PATCH 470/854] import ComponentsCommand correctly --- nf_core/components/list.py | 2 -- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/modules_test.py | 2 +- tests/modules/patch.py | 4 ++-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index eeacb9c39b..b51f8c4aa2 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -6,8 +6,6 @@ import nf_core.modules.modules_utils from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson - -# from .modules_command import ModulesRepo from nf_core.modules.modules_repo import ModulesRepo log = logging.getLogger(__name__) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 271c09e66b..dff57f9da5 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -21,8 +21,8 @@ import nf_core.modules.modules_utils import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.lint_utils import console -from nf_core.modules.modules_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson from nf_core.modules.nfcore_module import NFCoreModule from nf_core.utils import plural_s as _s diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index 1bcb4f5b74..97ef1b00f2 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -15,7 +15,7 @@ import nf_core.modules.modules_utils import nf_core.utils -from nf_core.modules.modules_command import ComponentCommand +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 4cd518c667..f1597cd776 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -4,8 +4,8 @@ import pytest +import nf_core.components.components_command import nf_core.modules -import nf_core.modules.modules_command from ..utils import GITLAB_URL @@ -300,7 +300,7 @@ def test_create_patch_update_fail(self): # Check that the installed files have not been affected by the attempted patch temp_dir = Path(tempfile.mkdtemp()) - nf_core.modules.modules_command.ComponentCommand( + nf_core.components.components_command.ComponentCommand( "modules", self.pipeline_dir, GITLAB_URL, PATCH_BRANCH ).install_component_files(BISMARK_ALIGN, FAIL_SHA, update_obj.modules_repo, temp_dir) From d2e5a7a935ea08f2812a591bdd20d3ccbf476f18 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:30:04 +0100 Subject: [PATCH 471/854] import ComponentsCommand correctly --- nf_core/subworkflows/test_yml_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index e7752ab5a2..271dd7edd5 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -25,9 +25,9 @@ from rich.syntax import Syntax import nf_core.utils +from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo -from nf_core.subworkflows.subworkflows_command import ComponentCommand log = logging.getLogger(__name__) From 56ff217353003294ba4546384017e6e81ba35026 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 4 Nov 2022 16:46:35 +0100 Subject: [PATCH 472/854] fix bug in remove --- nf_core/modules/remove.py | 4 ++-- tests/test_modules.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index afe16ca2d5..1930ecb879 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -12,8 +12,8 @@ class ModuleRemove(ComponentCommand): - def __init__(self): - super().__init__("modules") + def __init__(self, pipeline_dir): + super().__init__("modules", pipeline_dir) def remove(self, module): """ diff --git a/tests/test_modules.py b/tests/test_modules.py index 85e1047bf7..f14d256dc4 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -71,10 +71,18 @@ def setUp(self): branch=OLD_TRIMGALORE_BRANCH, ) self.mods_install_trimgalore = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + self.pipeline_dir, + prompt=False, + force=True, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, ) self.mods_install_gitlab = nf_core.modules.ModuleInstall( - self.pipeline_dir, prompt=False, force=True, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH + self.pipeline_dir, + prompt=False, + force=True, + remote_url=GITLAB_URL, + branch=GITLAB_DEFAULT_BRANCH, ) self.mods_install_gitlab_old = nf_core.modules.ModuleInstall( self.pipeline_dir, From af0bf3d9396d269ff4c139bcd786f58f8a60c7f5 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 5 Nov 2022 19:17:12 +0100 Subject: [PATCH 473/854] bump isort action version --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 89aa48462b..869d8898d9 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -81,7 +81,7 @@ jobs: with: python-version: 3.8 - name: python-isort - uses: isort/isort-action@v1.0.0 + uses: isort/isort-action@v1.1.0 with: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" From bc5e6f7d6578849bb7dac3db0e15184822fa384e Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 5 Nov 2022 19:21:01 +0100 Subject: [PATCH 474/854] updated GitHub actions --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e3448581..9fd4e4d330 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Run tests with Python 3.11 ([#1970](https://github.com/nf-core/tools/pull/1970)) - Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971)) - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out` +- Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001)) ### Modules From 1e46a5d023392f852eb60ba882d6cb3ec15a430b Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sat, 5 Nov 2022 21:37:47 +0100 Subject: [PATCH 475/854] simplify conditions and utilize lazy eval of elif --- nf_core/lint_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index bf9e9519ae..feb8dfac1c 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -64,12 +64,9 @@ def run_prettier_on_file(file): If Prettier is not installed, a warning is logged. """ - _prettier_installed = not shutil.which("prettier") is None - _pre_commit_installed = not shutil.which("pre-commit") is None - - if _prettier_installed: + if shutil.which("prettier"): _run_prettier_on_file(file) - elif _pre_commit_installed: + elif shutil.which("pre-commit"): _run_pre_commit_prettier_on_file(file) else: log.warning( From d648284ad9c908bdfe4d2a2a84505aa581f40db9 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Sun, 6 Nov 2022 10:46:04 +0100 Subject: [PATCH 476/854] fix typo in variable name --- nf_core/lint_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index feb8dfac1c..07b7f8ba21 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -45,7 +45,7 @@ def print_fixes(lint_obj, module_lint_obj): lint_dir = "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}" console.print( "\nTip: Some of these linting errors can automatically be resolved with the following command:" - f"\n\n[blue] nf-core lint {lint_dir} {fixe_flags}\n" + f"\n\n[blue] nf-core lint {lint_dir} {fix_flags}\n" ) if len(lint_obj.fix): console.print( From d143da64a5c51563cab78169e4e0977921431a7b Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 7 Nov 2022 08:02:32 +0100 Subject: [PATCH 477/854] remove unused argument --- nf_core/lint/__init__.py | 2 +- nf_core/lint_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 5ea01142e5..91391794c3 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -124,7 +124,7 @@ def run_linting( lint_obj._print_results(show_passed) module_lint_obj._print_results(show_passed) nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj) - nf_core.lint_utils.print_fixes(lint_obj, module_lint_obj) + nf_core.lint_utils.print_fixes(lint_obj) # Save results to Markdown file if md_fn is not None: diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 07b7f8ba21..1d2078ad94 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -36,7 +36,7 @@ def print_joint_summary(lint_obj, module_lint_obj): console.print(table) -def print_fixes(lint_obj, module_lint_obj): +def print_fixes(lint_obj): """Prints available and applied fixes""" fix_flags = " ".join([f"--fix {file}" for file in lint_obj.could_fix]) From dcb8ac05b56b5e4f78912dcc380f9b727c688445 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 7 Nov 2022 08:03:22 +0100 Subject: [PATCH 478/854] only compute fix flags if needed --- nf_core/lint_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 1d2078ad94..7f904b756c 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -39,10 +39,9 @@ def print_joint_summary(lint_obj, module_lint_obj): def print_fixes(lint_obj): """Prints available and applied fixes""" - fix_flags = " ".join([f"--fix {file}" for file in lint_obj.could_fix]) - - if fix_flags: + if lint_obj.could_fix: lint_dir = "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}" + fix_flags = " ".join([f"--fix {file}" for file in lint_obj.could_fix]) console.print( "\nTip: Some of these linting errors can automatically be resolved with the following command:" f"\n\n[blue] nf-core lint {lint_dir} {fix_flags}\n" From bea0218f5442d92d4d20ddda1d5ff20b7c4a4aef Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 10:24:23 +0100 Subject: [PATCH 479/854] run isort --- nf_core/modules/install.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 65cd9773bc..2a38f16c4c 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -4,9 +4,8 @@ import nf_core.components.components_install import nf_core.modules.modules_utils import nf_core.utils -from nf_core.modules.modules_json import ModulesJson - from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) From 7c93bd59f63954678142a0d5eb9f2e658fc71bb9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 10:51:12 +0100 Subject: [PATCH 480/854] add component_type to ComponentsTest --- nf_core/components/components_test.py | 21 +++- nf_core/modules/modules_test.py | 172 ++------------------------ 2 files changed, 29 insertions(+), 164 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 7c6a9be649..69218e5e5d 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -1,4 +1,19 @@ +import logging +import os +import sys +from pathlib import Path +from shutil import which + +import pytest +import questionary +import rich + +import nf_core.modules.modules_utils +import nf_core.utils from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson + +log = logging.getLogger(__name__) class ComponentsTest(ComponentCommand): @@ -30,6 +45,7 @@ class ComponentsTest(ComponentCommand): def __init__( self, + component_type, module_name=None, no_prompts=False, pytest_args="", @@ -37,12 +53,11 @@ def __init__( branch=None, no_pull=False, ): + super().__init__(component_type=component_type, dir=".", remote_url=remote_url, branch=branch, no_pull=no_pull) self.module_name = module_name self.no_prompts = no_prompts self.pytest_args = pytest_args - super().__init__("component_type", ".", remote_url, branch, no_pull) - def run(self): """Run test steps""" if not self.no_prompts: @@ -61,7 +76,7 @@ def _check_inputs(self): # Retrieving installed modules if self.repo_type == "modules": - installed_modules = self.get_modules_clone_modules() + installed_modules = self.get_components_clone_modules() else: modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index 97ef1b00f2..f02ea11cdb 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -3,49 +3,12 @@ The ModulesTest class runs the tests locally """ -import logging -import os -import sys -from pathlib import Path -from shutil import which +from nf_core.components.components_test import ComponentsTest -import pytest -import questionary -import rich -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson - -log = logging.getLogger(__name__) - - -class ModulesTest(ComponentCommand): +class ModulesTest(ComponentsTest): """ Class to run module pytests. - - ... - - Attributes - ---------- - module_name : str - name of the tool to run tests for - no_prompts : bool - flat indicating if prompts are used - pytest_args : tuple - additional arguments passed to pytest command - - Methods - ------- - run(): - Run test steps - _check_inputs(): - Check inputs. Ask for module_name if not provided and check that the directory exists - _set_profile(): - Set software profile - _run_pytests(self): - Run pytest """ def __init__( @@ -57,125 +20,12 @@ def __init__( branch=None, no_pull=False, ): - self.module_name = module_name - self.no_prompts = no_prompts - self.pytest_args = pytest_args - - super().__init__("modules", ".", remote_url, branch, no_pull) - - def run(self): - """Run test steps""" - if not self.no_prompts: - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" - ) - self._check_inputs() - self._set_profile() - self._check_profile() - self._run_pytests() - - def _check_inputs(self): - """Do more complex checks about supplied flags.""" - # Check modules directory structure - self.check_modules_structure() - - # Retrieving installed modules - if self.repo_type == "modules": - installed_modules = self.get_components_clone_modules() - else: - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url) - - # Get the tool name if not specified - if self.module_name is None: - if self.no_prompts: - raise UserWarning( - "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." - ) - if not installed_modules: - raise UserWarning( - f"No installed modules were found from '{self.modules_repo.remote_url}'.\n" - f"Are you running the tests inside the nf-core/modules main directory?\n" - f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/" - ) - self.module_name = questionary.autocomplete( - "Tool name:", - choices=installed_modules, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Sanity check that the module directory exists - self._validate_folder_structure() - - def _validate_folder_structure(self): - """Validate that the modules follow the correct folder structure to run the tests: - - modules/nf-core/TOOL/SUBTOOL/ - - tests/modules/nf-core/TOOL/SUBTOOL/ - - """ - module_path = Path(self.default_modules_path) / self.module_name - test_path = Path(self.default_tests_path) / self.module_name - - if not (self.dir / module_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" - ) - if not (self.dir / test_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. " - "Are you running the tests inside the nf-core/modules main directory? " - "Do you have tests for the specified module?" - ) - - def _set_profile(self): - """Set $PROFILE env variable. - The config expects $PROFILE and Nextflow fails if it's not set. - """ - if os.environ.get("PROFILE") is None: - os.environ["PROFILE"] = "" - if self.no_prompts: - log.info( - "Setting environment variable '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " - "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." - ) - else: - question = { - "type": "list", - "name": "profile", - "message": "Choose software profile", - "choices": ["Docker", "Singularity", "Conda"], - } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() - os.environ["PROFILE"] = profile - log.info(f"Setting environment variable '$PROFILE' to '{profile}'") - - def _check_profile(self): - """Check if profile is available""" - profile = os.environ.get("PROFILE") - # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "conda"] - if profile in valid_nextflow_profiles: - if not which(profile): - raise UserWarning(f"Command '{profile}' not found - is it installed?") - else: - raise UserWarning( - f"The PROFILE '{profile}' set in the shell environment is not valid.\n" - f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." - ) - - def _run_pytests(self): - """Given a module name, run tests.""" - # Print nice divider line - console = rich.console.Console() - console.rule(self.module_name, style="black") - - # Set pytest arguments - command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] - command_args += self.pytest_args - - # Run pytest - log.info(f"Running pytest for module '{self.module_name}'") - sys.exit(pytest.main(command_args)) + super().__init__( + component_type="modules", + module_name=module_name, + no_prompts=no_prompts, + pytest_args=pytest_args, + remote_url=remote_url, + branch=branch, + no_pull=no_pull, + ) From 387206242f2d4f0e183b806c86ceaa5df8307329 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 13:34:44 +0100 Subject: [PATCH 481/854] modules_json create() adds subworkflows --- nf_core/modules/modules_json.py | 236 +++++++++++++++++++------------- 1 file changed, 138 insertions(+), 98 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 151914b38b..5f1ac55094 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -42,7 +42,7 @@ def __init__(self, pipeline_dir): def create(self): """ - Creates the modules.json file from the modules installed in the pipeline directory + Creates the modules.json file from the modules and subworkflows installed in the pipeline directory Raises: UserWarning: If the creation fails @@ -51,25 +51,60 @@ def create(self): pipeline_name = pipeline_config.get("manifest.name", "") pipeline_url = pipeline_config.get("manifest.homePage", "") modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": {}} - modules_dir = Path(self.dir, "modules") - if not modules_dir.exists(): + if not self.modules_dir.exists(): raise UserWarning("Can't find a ./modules directory. Is this a DSL2 pipeline?") - repos, _ = self.get_pipeline_module_repositories(modules_dir) + # Get repositories + repos, _ = self.get_pipeline_module_repositories("modules", self.modules_dir) + repos, _ = self.get_pipeline_module_repositories("subworkflows", self.subworkflows_dir, repos) - # Get all module names in the repos - repo_module_names = [ + # Get all module/subworkflow names in the repos + repo_module_names = self.get_component_names_from_repo(repos, self.modules_dir) + repo_subworkflow_names = self.get_component_names_from_repo(repos, self.subworkflows_dir) + + # Add module/subworkflow info into modules_json + for repo_url, module_names, install_dir in sorted(repo_module_names): + modules_json["repos"][repo_url] = {} + modules_json["repos"][repo_url]["modules"] = {} + modules_json["repos"][repo_url]["modules"][install_dir] = {} + modules_json["repos"][repo_url]["modules"][install_dir] = self.determine_branches_and_shas( + "modules", install_dir, repo_url, module_names + ) + for repo_url, subworkflow_names, install_dir in sorted(repo_subworkflow_names): + modules_json["repos"][repo_url] = {} + modules_json["repos"][repo_url]["subworkflows"] = {} + modules_json["repos"][repo_url]["subworkflows"][install_dir] = {} + modules_json["repos"][repo_url]["subworkflows"][install_dir] = self.determine_branches_and_shas( + "subworkflows", install_dir, repo_url, subworkflow_names + ) + + # write the modules.json file and assign it to the object + modules_json_path = Path(self.dir, "modules.json") + with open(modules_json_path, "w") as fh: + json.dump(modules_json, fh, indent=4) + fh.write("\n") + self.modules_json = modules_json + + def get_component_names_from_repo(self, repos, directory): + """ + Get component names from repositories in a pipeline. + + Args: + repos (list): list of repository urls + directory (str): modules directory or subworkflows directory + """ + names = [ ( repo_url, [ str( - Path(module_name).relative_to( - modules_dir / nf_core.modules.modules_utils.path_from_remote(repo_url) + Path(component_name).relative_to( + directory / nf_core.modules.modules_utils.path_from_remote(repo_url) ) ) - for module_name, _, file_names in os.walk( - modules_dir / nf_core.modules.modules_utils.path_from_remote(repo_url) + for component_name, _, file_names in os.walk( + directory / nf_core.modules.modules_utils.path_from_remote(repo_url) ) if "main.nf" in file_names ], @@ -77,28 +112,16 @@ def create(self): ) for repo_url in repos ] + return names - for repo_url, module_names, install_dir in sorted(repo_module_names): - modules_json["repos"][repo_url] = {} - modules_json["repos"][repo_url]["modules"] = {} - modules_json["repos"][repo_url]["modules"][install_dir] = {} - modules_json["repos"][repo_url]["modules"][install_dir] = self.determine_module_branches_and_shas( - install_dir, repo_url, module_names - ) - # write the modules.json file and assign it to the object - modules_json_path = Path(self.dir, "modules.json") - with open(modules_json_path, "w") as fh: - json.dump(modules_json, fh, indent=4) - fh.write("\n") - self.modules_json = modules_json - - def get_pipeline_module_repositories(self, modules_dir, repos=None): + def get_pipeline_module_repositories(self, component_type, directory, repos=None): """ - Finds all module repositories in the modules directory. - Ignores the local modules. + Finds all module repositories in the modules and subworkflows directory. + Ignores the local modules/subworkflows. Args: - modules_dir (Path): base directory for the module files + component_type (str): modules or subworkflows + directory (Path): base directory for the module files Returns repos ([ (str, str, str) ]), renamed_dirs (dict[Path, Path]): List of tuples of repo name, repo @@ -108,22 +131,23 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): if repos is None: repos = {} # Check if there are any nf-core modules installed - if (modules_dir / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): + if (directory / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): repos[nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE] = {} # The function might rename some directories, keep track of them renamed_dirs = {} # Check if there are any untracked repositories dirs_not_covered = self.dir_tree_uncovered( - modules_dir, [Path(nf_core.modules.modules_utils.path_from_remote(url)) for url in repos] + directory, [Path(nf_core.modules.modules_utils.path_from_remote(url)) for url in repos] ) if len(dirs_not_covered) > 0: - log.info("Found custom module repositories when creating 'modules.json'") + log.info(f"Found custom {component_type[:-1]} repositories when creating 'modules.json'") # Loop until all directories in the base directory are covered by a remote while len(dirs_not_covered) > 0: log.info( - "The following director{s} in the modules directory are untracked: '{l}'".format( + "The following director{s} in the {t} directory are untracked: '{l}'".format( s="ies" if len(dirs_not_covered) > 0 else "y", - l="', '".join(str(dir.relative_to(modules_dir)) for dir in dirs_not_covered), + t=component_type, + l="', '".join(str(dir.relative_to(directory)) for dir in dirs_not_covered), ) ) nrepo_remote = questionary.text( @@ -142,7 +166,7 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): # Verify that there is a directory corresponding the remote nrepo_name = nf_core.modules.modules_utils.path_from_remote(nrepo_remote) - if not (modules_dir / nrepo_name).exists(): + if not (directory / nrepo_name).exists(): log.info( "The provided remote does not seem to correspond to a local directory. " "The directory structure should be the same as in the remote." @@ -152,28 +176,28 @@ def get_pipeline_module_repositories(self, modules_dir, repos=None): style=nf_core.utils.nfcore_question_style, ).unsafe_ask() if dir_name: - old_path = modules_dir / dir_name - new_path = modules_dir / nrepo_name + old_path = directory / dir_name + new_path = directory / nrepo_name old_path.rename(new_path) renamed_dirs[old_path] = new_path else: continue - repos[nrepo_remote]["modules"][nrepo_name] = {} + repos[nrepo_remote][component_type][nrepo_name] = {} dirs_not_covered = self.dir_tree_uncovered( - modules_dir, [Path(name) for url in repos for name in repos[url][modules_dir]] + directory, [Path(name) for url in repos for name in repos[url][directory]] ) return repos, renamed_dirs - def dir_tree_uncovered(self, modules_dir, repos): + def dir_tree_uncovered(self, directory, repos): """ - Does a BFS of the modules directory to look for directories that + Does a BFS of the modules/subworkflos directory to look for directories that are not tracked by a remote. The 'repos' argument contains the directories that are currently covered by remote, and it and its subdirectories are therefore ignore. Args: - module_dir (Path): Base path of modules in pipeline + directory (Path): Base path of modules or subworkflows in pipeline repos ([ Path ]): List of repos that are covered by a remote Returns: @@ -181,14 +205,14 @@ def dir_tree_uncovered(self, modules_dir, repos): """ # Initialise the FIFO queue. Note that we assume the directory to be correctly # configured, i.e. no files etc. - fifo = [subdir for subdir in modules_dir.iterdir() if subdir.stem != "local"] + fifo = [subdir for subdir in directory.iterdir() if subdir.stem != "local"] depth = 1 dirs_not_covered = [] while len(fifo) > 0: temp_queue = [] repos_at_level = {Path(*repo.parts[:depth]): len(repo.parts) for repo in repos} for directory in fifo: - rel_dir = directory.relative_to(modules_dir) + rel_dir = directory.relative_to(directory) if rel_dir in repos_at_level.keys(): # Go the next depth if this directory is not one of the repos if depth < repos_at_level[rel_dir]: @@ -200,61 +224,70 @@ def dir_tree_uncovered(self, modules_dir, repos): depth += 1 return dirs_not_covered - def determine_module_branches_and_shas(self, install_dir, remote_url, modules): + def determine_branches_and_shas(self, component_type, install_dir, remote_url, components): """ - Determines what branch and commit sha each module in the pipeline belong to + Determines what branch and commit sha each module/subworkflow in the pipeline belong to - Assumes all modules are installed from the default branch. If it fails to find the - module in the default branch, it prompts the user with the available branches + Assumes all modules/subworkflows are installed from the default branch. If it fails to find the + module/subworkflow in the default branch, it prompts the user with the available branches Args: - install_dir (str): The name of the directory inside modules where modules are installed + install_dir (str): The name of the directory inside modules or subworkflows where components are installed remote_url (str): The url to the remote repository - modules ([str]): List of names of installed modules from the repository + components ([str]): List of names of installed modules/subworkflows from the repository Returns: - (dict[str, dict[str, str]]): The module.json entries for the modules + (dict[str, dict[str, str]]): The module.json entries for the modules/subworkflows from the repository """ default_modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=remote_url) - repo_path = self.modules_dir / install_dir + if component_type == "modules": + repo_path = self.modules_dir / install_dir + elif component_type == "subworkflows": + repo_path = self.subworkflows_dir / install_dir # Get the branches present in the repository, as well as the default branch available_branches = nf_core.modules.modules_repo.ModulesRepo.get_remote_branches(remote_url) sb_local = [] - dead_modules = [] + dead_components = [] repo_entry = {} - for module in sorted(modules): + for component in sorted(components): modules_repo = default_modules_repo - module_path = repo_path / module + component_path = repo_path / component correct_commit_sha = None tried_branches = {default_modules_repo.branch} found_sha = False while True: - # If the module is patched - patch_file = module_path / f"{module}.diff" + # If the module/subworkflow is patched + patch_file = component_path / f"{component}.diff" if patch_file.is_file(): - temp_module_dir = self.try_apply_patch_reverse(module, install_dir, patch_file, module_path) - correct_commit_sha = self.find_correct_commit_sha(module, temp_module_dir, modules_repo) + temp_module_dir = self.try_apply_patch_reverse(component, install_dir, patch_file, component_path) + correct_commit_sha = self.find_correct_commit_sha( + component_type, component, temp_module_dir, modules_repo + ) else: - correct_commit_sha = self.find_correct_commit_sha(module, module_path, modules_repo) + correct_commit_sha = self.find_correct_commit_sha( + component_type, component, component_path, modules_repo + ) if correct_commit_sha is None: # Check in the old path correct_commit_sha = self.find_correct_commit_sha( - module, repo_path / "modules" / module, modules_repo + component_type, component, repo_path / component_type / component, modules_repo ) if correct_commit_sha is None: - log.info(f"Was unable to find matching module files in the {modules_repo.branch} branch.") + log.info( + f"Was unable to find matching {component_type[:-1]} files in the {modules_repo.branch} branch." + ) choices = [{"name": "No", "value": False}] + [ {"name": branch, "value": branch} for branch in (available_branches - tried_branches) ] branch = questionary.select( - f"Was the module '{module}' installed from a different branch in the remote?\nSelect 'No' for a local module", + f"Was the {component_type[:-1]} '{component}' installed from a different branch in the remote?\nSelect 'No' for a local {component_type[:-1]}", choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() if not branch: action = questionary.select( - f"Module is untracked '{module}'. Please select what action to take", + f"{component_type[:-1].title()} is untracked '{component}'. Please select what action to take", choices=[ {"name": "Move the directory to 'local'", "value": 0}, {"name": "Remove the files", "value": 1}, @@ -262,9 +295,9 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): style=nf_core.utils.nfcore_question_style, ).unsafe_ask() if action == 0: - sb_local.append(module) + sb_local.append(component) else: - dead_modules.append(module) + dead_components.append(component) break # Create a new modules repo with the selected branch, and retry find the sha modules_repo = nf_core.modules.modules_repo.ModulesRepo( @@ -274,61 +307,67 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): found_sha = True break if found_sha: - repo_entry[module] = {"branch": modules_repo.branch, "git_sha": correct_commit_sha} + repo_entry[component] = {"branch": modules_repo.branch, "git_sha": correct_commit_sha} - # Clean up the modules we were unable to find the sha for - for module in sb_local: - log.debug(f"Moving module '{Path(install_dir, module)}' to 'local' directory") - self.move_module_to_local(module, install_dir) + # Clean up the modules/subworkflows we were unable to find the sha for + for component in sb_local: + log.debug(f"Moving {component_type[:-1]} '{Path(install_dir, component)}' to 'local' directory") + self.move_module_to_local(component, install_dir) - for module in dead_modules: - log.debug(f"Removing module {Path(install_dir, module)}'") - shutil.rmtree(repo_path / module) + for component in dead_components: + log.debug(f"Removing {component_type[:-1]} {Path(install_dir, component)}'") + shutil.rmtree(repo_path / component) return repo_entry - def find_correct_commit_sha(self, module_name, module_path, modules_repo): + def find_correct_commit_sha(self, component_type, component_name, component_path, modules_repo): """ Returns the SHA for the latest commit where the local files are identical to the remote files Args: - module_name (str): Name of module - module_path (str): Path to module in local repo - module_repo (str): Remote repo for module + component_type (str): modules or subworkflows + component_name (str): Name of module/subowrkflow + component_path (str): Path to module/subworkflow in local repo + modules_repo (str): Remote repo for module/subworkflow Returns: commit_sha (str): The latest commit SHA where local files are identical to remote files, or None if no commit is found """ - # Find the correct commit SHA for the local module files. - # We iterate over the commit history for the module until we find + # Find the correct commit SHA for the local module/subworkflow files. + # We iterate over the commit history for the module/subworkflow until we find # a revision that matches the file contents commit_shas = ( - commit["git_sha"] for commit in modules_repo.get_component_git_log(module_name, "modules", depth=1000) + commit["git_sha"] + for commit in modules_repo.get_component_git_log(component_name, component_type, depth=1000) ) for commit_sha in commit_shas: - if all(modules_repo.module_files_identical(module_name, module_path, commit_sha).values()): + if all(modules_repo.module_files_identical(component_name, component_path, commit_sha).values()): return commit_sha return None - def move_module_to_local(self, module, repo_name): + def move_component_to_local(self, component_type, component, repo_name): """ - Move a module to the 'local' directory + Move a module/subworkflow to the 'local' directory Args: - module (str): The name of the modules + component (str): The name of the module/subworkflow repo_name (str): The name of the repository the module resides in """ - current_path = self.modules_dir / repo_name / module - local_modules_dir = self.modules_dir / "local" - if not local_modules_dir.exists(): - local_modules_dir.mkdir() + if component_type == "modules": + directory = self.modules_dir + elif component_type == "subworkflows": + directory = self.subworkflows_dir + current_path = directory / repo_name / component + local_dir = directory / "local" + if not local_dir.exists(): + local_dir.mkdir() - to_name = module + to_name = component # Check if there is already a subdirectory with the name - while (local_modules_dir / to_name).exists(): + while (local_dir / to_name).exists(): # Add a time suffix to the path to make it unique # (do it again and again if it didn't work out...) to_name += f"-{datetime.datetime.now().strftime('%y%m%d%H%M%S')}" - shutil.move(current_path, local_modules_dir / to_name) + shutil.move(current_path, local_dir / to_name) def unsynced_components(self): """ @@ -379,17 +418,18 @@ def parse_dirs(self, dirs, missing_installation, component_type): git_url = repo break if not component_in_file: - # If it is not, add it to the list of missing subworkflow + # If it is not, add it to the list of missing modules/subworkflows untracked_dirs.append(component) - else: - # If it does, remove the subworkflow from missing_installation + # If it does, remove the module/subworkflow from missing_installation module_repo = missing_installation[git_url] # Check if the entry has a git sha and branch before removing components_dict = module_repo[component_type][install_dir] if "git_sha" not in components_dict[component] or "branch" not in components_dict[component]: - self.determine_module_branches_and_shas(component, git_url, module_repo["base_path"], [component]) - # Remove the subworkflow from subworkflows without installation + self.determine_branches_and_shas( + component_type, component, git_url, module_repo["base_path"], [component] + ) + # Remove the module/subworkflow from modules/subworkflows without installation module_repo[component_type][install_dir].pop(component) if len(module_repo[component_type][install_dir]) == 0: # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation @@ -921,7 +961,7 @@ def resolve_missing_from_modules_json(self, missing_from_modules_json, component # Get the remotes we are missing tracked_repos = {repo_url: (repo_entry) for repo_url, repo_entry in self.modules_json["repos"].items()} - repos, _ = self.get_pipeline_module_repositories(self.modules_dir, tracked_repos) + repos, _ = self.get_pipeline_module_repositories("modules", self.modules_dir, tracked_repos) components_with_repos = ( ( @@ -941,7 +981,7 @@ def resolve_missing_from_modules_json(self, missing_from_modules_json, component for install_dir, components in repos_with_components.items(): remote_url = [url for url, content in repos.items() if install_dir in content][0] - repo_entry = self.determine_module_branches_and_shas(install_dir, remote_url, components) + repo_entry = self.determine_branches_and_shas("component_type", install_dir, remote_url, components) if remote_url in self.modules_json["repos"]: self.modules_json["repos"][remote_url][component_type][install_dir].update(repo_entry) else: From 0ec76e96141bac2517f8154ad7b79a63e9410c02 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 13:36:02 +0100 Subject: [PATCH 482/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd4e4d330..6e543db5f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - Add tests for subworkflows install command ([#1996](https://github.com/nf-core/tools/pull/1996)) - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) +- Function create() from modules_json.py adds also subworkflows to modules.json file () ## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] From 80cd6de39c13ad563172aff333b24dec44b31097 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 14:02:31 +0100 Subject: [PATCH 483/854] modify changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2b8a9d70b..d6f865f18b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ - Add tests for subworkflows install command ([#1996](https://github.com/nf-core/tools/pull/1996)) - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) -- Function create() from modules_json.py adds also subworkflows to modules.json file () +- Function create() from modules_json.py adds also subworkflows to modules.json file ([#2005](https://github.com/nf-core/tools/pull/2005)) ## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] From 1de5fc5a2cd0c8f3f03030926e05cd452127f8cb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 16:29:47 +0100 Subject: [PATCH 484/854] we add installed subworkflows to modules.json --- nf_core/modules/modules_json.py | 38 ++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index da5c29fbeb..0eb4d9285f 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -131,7 +131,9 @@ def get_pipeline_module_repositories(self, component_type, directory, repos=None if repos is None: repos = {} # Check if there are any nf-core modules installed - if (directory / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME).exists(): + if ( + directory / nf_core.modules.modules_repo.NF_CORE_MODULES_NAME + ).exists() and nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE not in repos.keys(): repos[nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE] = {} # The function might rename some directories, keep track of them renamed_dirs = {} @@ -189,7 +191,7 @@ def get_pipeline_module_repositories(self, component_type, directory, repos=None ) return repos, renamed_dirs - def dir_tree_uncovered(self, directory, repos): + def dir_tree_uncovered(self, components_directory, repos): """ Does a BFS of the modules/subworkflos directory to look for directories that are not tracked by a remote. The 'repos' argument contains the @@ -197,7 +199,7 @@ def dir_tree_uncovered(self, directory, repos): subdirectories are therefore ignore. Args: - directory (Path): Base path of modules or subworkflows in pipeline + components_directory (Path): Base path of modules or subworkflows in pipeline repos ([ Path ]): List of repos that are covered by a remote Returns: @@ -205,14 +207,14 @@ def dir_tree_uncovered(self, directory, repos): """ # Initialise the FIFO queue. Note that we assume the directory to be correctly # configured, i.e. no files etc. - fifo = [subdir for subdir in directory.iterdir() if subdir.stem != "local"] + fifo = [subdir for subdir in components_directory.iterdir() if subdir.stem != "local"] depth = 1 dirs_not_covered = [] while len(fifo) > 0: temp_queue = [] repos_at_level = {Path(*repo.parts[:depth]): len(repo.parts) for repo in repos} for directory in fifo: - rel_dir = directory.relative_to(directory) + rel_dir = directory.relative_to(components_directory) if rel_dir in repos_at_level.keys(): # Go the next depth if this directory is not one of the repos if depth < repos_at_level[rel_dir]: @@ -1012,18 +1014,29 @@ def resolve_missing_from_modules_json(self, missing_from_modules_json, component # Get the remotes we are missing tracked_repos = {repo_url: (repo_entry) for repo_url, repo_entry in self.modules_json["repos"].items()} - repos, _ = self.get_pipeline_module_repositories("modules", self.modules_dir, tracked_repos) + repos, _ = self.get_pipeline_module_repositories(component_type, self.modules_dir, tracked_repos) + # Get tuples of components that miss installation and their install directory components_with_repos = ( ( nf_core.modules.modules_utils.path_from_remote(repo_url), - str(dir.relative_to(nf_core.modules.modules_utils.path_from_remote(repo_url))), + dir, ) for dir in missing_from_modules_json for repo_url in repos - if nf_core.utils.is_relative_to(dir, nf_core.modules.modules_utils.path_from_remote(repo_url)) + if dir + in [ + Path(x[0]).parts[-1] + for x in os.walk( + Path( + self.modules_dir if component_type == "modules" else self.subworkflows_dir, + nf_core.modules.modules_utils.path_from_remote(repo_url), + ) + ) + ] ) + # Add all components into a dictionary with install directories repos_with_components = {} for install_dir, component in components_with_repos: if install_dir not in repos_with_components: @@ -1031,8 +1044,13 @@ def resolve_missing_from_modules_json(self, missing_from_modules_json, component repos_with_components[install_dir].append(component) for install_dir, components in repos_with_components.items(): - remote_url = [url for url, content in repos.items() if install_dir in content][0] - repo_entry = self.determine_branches_and_shas("component_type", install_dir, remote_url, components) + remote_url = [ + url + for url, content in repos.items() + for comp_type, install_directories in content.items() + if install_dir in install_directories + ][0] + repo_entry = self.determine_branches_and_shas(component_type, install_dir, remote_url, components) if remote_url in self.modules_json["repos"]: self.modules_json["repos"][remote_url][component_type][install_dir].update(repo_entry) else: From 2b058a628a58287e97f6f983a1d2742ac9158748 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 7 Nov 2022 17:23:40 +0100 Subject: [PATCH 485/854] fix bug with modules --- nf_core/modules/modules_json.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 0eb4d9285f..4af9465806 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1017,28 +1017,26 @@ def resolve_missing_from_modules_json(self, missing_from_modules_json, component repos, _ = self.get_pipeline_module_repositories(component_type, self.modules_dir, tracked_repos) # Get tuples of components that miss installation and their install directory - components_with_repos = ( - ( - nf_core.modules.modules_utils.path_from_remote(repo_url), - dir, - ) - for dir in missing_from_modules_json - for repo_url in repos - if dir - in [ - Path(x[0]).parts[-1] - for x in os.walk( - Path( - self.modules_dir if component_type == "modules" else self.subworkflows_dir, + def components_with_repos(): + for dir in missing_from_modules_json: + for repo_url in repos: + paths_in_directory = [] + repo_url_path = Path( + self.modules_dir, nf_core.modules.modules_utils.path_from_remote(repo_url), ) - ) - ] - ) + for dir_name, _, _ in os.walk(repo_url_path): + if component_type == "modules": + if len(Path(dir).parts) > 1: # The module name is TOOL/SUBTOOL + paths_in_directory.append(str(Path(*Path(dir_name).parts[-2:]))) + pass + paths_in_directory.append(Path(dir_name).parts[-1]) + if dir in paths_in_directory: + yield (nf_core.modules.modules_utils.path_from_remote(repo_url), dir) # Add all components into a dictionary with install directories repos_with_components = {} - for install_dir, component in components_with_repos: + for install_dir, component in components_with_repos(): if install_dir not in repos_with_components: repos_with_components[install_dir] = [] repos_with_components[install_dir].append(component) From dadb11c2a42a5011fb91716e5d10dfe43b00d65a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 09:38:01 +0100 Subject: [PATCH 486/854] change installed by installed_by --- nf_core/components/components_install.py | 2 +- nf_core/modules/modules_json.py | 26 ++++++++++++------------ nf_core/pipeline-template/modules.json | 6 +++--- tests/modules/modules_json.py | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 7680134d14..5ad00695bc 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -100,4 +100,4 @@ def clean_modules_json(component, component_type, modules_repo, modules_json): f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" ) modules_json.remove_entry(component_type, component, repo_to_remove, modules_repo.repo_path) - return component_values["installed"] + return component_values["installed_by"] diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index eabaa51c73..d643057781 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -277,7 +277,7 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules): repo_entry[module] = { "branch": modules_repo.branch, "git_sha": correct_commit_sha, - "installed": "modules", + "installed_by": "modules", } # Clean up the modules we were unable to find the sha for @@ -481,7 +481,7 @@ def check_up_to_date(self): If a module/subworkflow is installed but the entry in 'modules.json' is missing we iterate through the commit log in the remote to try to determine the SHA. - Check that we have the "installed" value in 'modules.json', otherwise add it. + Check that we have the "installed_by" value in 'modules.json', otherwise add it. Assume that the modules/subworkflows were installed by and nf-core command (don't track installed by subworkflows). """ try: @@ -517,13 +517,13 @@ def check_up_to_date(self): if len(subworkflows_missing_from_modules_json) > 0: self.resolve_missing_from_modules_json(subworkflows_missing_from_modules_json, "subworkflows") - # If the "installed" value is not present for modules/subworkflows, add it. + # If the "installed_by" value is not present for modules/subworkflows, add it. for repo, repo_content in self.modules_json["repos"].items(): for component_type, dir_content in repo_content.items(): for install_dir, installed_components in dir_content.items(): for component, component_features in installed_components.items(): - if "installed" not in component_features: - self.modules_json["repos"][repo][component_type][install_dir][component]["installed"] = [ + if "installed_by" not in component_features: + self.modules_json["repos"][repo][component_type][install_dir][component]["installed_by"] = [ component_type ] @@ -572,12 +572,12 @@ def update(self, modules_repo, module_name, module_version, installed_by, instal repo_modules_entry[module_name]["git_sha"] = module_version repo_modules_entry[module_name]["branch"] = branch try: - if installed_by not in repo_modules_entry[module_name]["installed"]: - repo_modules_entry[module_name]["installed"].append(installed_by) + if installed_by not in repo_modules_entry[module_name]["installed_by"]: + repo_modules_entry[module_name]["installed_by"].append(installed_by) except KeyError: - repo_modules_entry[module_name]["installed"] = [installed_by] + repo_modules_entry[module_name]["installed_by"] = [installed_by] finally: - repo_modules_entry[module_name]["installed"].extend(installed_by_log) + repo_modules_entry[module_name]["installed_by"].extend(installed_by_log) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) @@ -616,12 +616,12 @@ def update_subworkflow( repo_subworkflows_entry[subworkflow_name]["git_sha"] = subworkflow_version repo_subworkflows_entry[subworkflow_name]["branch"] = branch try: - if installed_by not in repo_subworkflows_entry[subworkflow_name]["installed"]: - repo_subworkflows_entry[subworkflow_name]["installed"].append(installed_by) + if installed_by not in repo_subworkflows_entry[subworkflow_name]["installed_by"]: + repo_subworkflows_entry[subworkflow_name]["installed_by"].append(installed_by) except KeyError: - repo_subworkflows_entry[subworkflow_name]["installed"] = [installed_by] + repo_subworkflows_entry[subworkflow_name]["installed_by"] = [installed_by] finally: - repo_subworkflows_entry[subworkflow_name]["installed"].extend(installed_by_log) + repo_subworkflows_entry[subworkflow_name]["installed_by"].extend(installed_by_log) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 6a1bf1f96a..8618bacab6 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -8,17 +8,17 @@ "custom/dumpsoftwareversions": { "branch": "master", "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", - "installed": ["modules"] + "installed_by": ["modules"] }, "fastqc": { "branch": "master", "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", - "installed": ["modules"] + "installed_by": ["modules"] }, "multiqc": { "branch": "master", "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", - "installed": ["modules"] + "installed_by": ["modules"] } } } diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 3b6be3101f..20eee54e30 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -41,7 +41,7 @@ def test_mod_json_update(self): NF_CORE_MODULES_DEFAULT_BRANCH == mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["branch"] ) - assert "modules" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["installed"] + assert "modules" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"]["installed_by"] def test_mod_json_create(self): From a5073fa6b9fb5dcb17466e46d2aa2dd002cf7f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 8 Nov 2022 10:22:17 +0100 Subject: [PATCH 487/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index d643057781..e2c45b935b 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -482,7 +482,7 @@ def check_up_to_date(self): the commit log in the remote to try to determine the SHA. Check that we have the "installed_by" value in 'modules.json', otherwise add it. - Assume that the modules/subworkflows were installed by and nf-core command (don't track installed by subworkflows). + Assume that the modules/subworkflows were installed by an nf-core command (don't track installed by subworkflows). """ try: self.load() From ee7a6798fdc2ac3de45fd5087ea641db4af1a9bb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 11:20:07 +0100 Subject: [PATCH 488/854] don't overwrite repo when adding subworkflows --- nf_core/modules/modules_json.py | 4 ++-- tests/modules/modules_json.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 32c9ff4946..9fc0275163 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -57,7 +57,6 @@ def create(self): # Get repositories repos, _ = self.get_pipeline_module_repositories("modules", self.modules_dir) - repos, _ = self.get_pipeline_module_repositories("subworkflows", self.subworkflows_dir, repos) # Get all module/subworkflow names in the repos repo_module_names = self.get_component_names_from_repo(repos, self.modules_dir) @@ -72,7 +71,8 @@ def create(self): "modules", install_dir, repo_url, module_names ) for repo_url, subworkflow_names, install_dir in sorted(repo_subworkflow_names): - modules_json["repos"][repo_url] = {} + if repo_url not in modules_json["repos"]: # Don't overwrite the repo if it was already added by modules + modules_json["repos"][repo_url] = {} modules_json["repos"][repo_url]["subworkflows"] = {} modules_json["repos"][repo_url]["subworkflows"][install_dir] = {} modules_json["repos"][repo_url]["subworkflows"][install_dir] = self.determine_branches_and_shas( diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 20eee54e30..8ad7c77b5c 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -60,6 +60,7 @@ def test_mod_json_create(self): # Get the contents of the file mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() + print(mod_json) mods = ["fastqc", "multiqc"] for mod in mods: From 9fd155631242001bb27af6cf85c20f42a56f86e6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 13:29:17 +0100 Subject: [PATCH 489/854] SubworkflowsTest inherits ComponentsTest (not functional) --- nf_core/subworkflows/subworkflows_test.py | 169 ++-------------------- 1 file changed, 11 insertions(+), 158 deletions(-) diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 6ecf3aebf7..803e808bde 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -3,49 +3,12 @@ The SubworkflowsTest class runs the tests locally """ -import logging -import os -import sys -from pathlib import Path -from shutil import which +from nf_core.components.components_test import ComponentsTest -import pytest -import questionary -import rich -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson - -log = logging.getLogger(__name__) - - -class SubworkflowsTest(ComponentCommand): +class SubworkflowsTest(ComponentsTest): """ Class to run module pytests. - - ... - - Attributes - ---------- - subworkflow_name : str - name of the subworkflow to run tests for - no_prompts : bool - flat indicating if prompts are used - pytest_args : tuple - additional arguments passed to pytest command - - Methods - ------- - run(): - Run test steps - _check_inputs(): - Check inputs. Ask for subworkflow_name if not provided and check that the directory exists - _set_profile(): - Set software profile - _run_pytests(self): - Run pytest """ def __init__( @@ -57,122 +20,12 @@ def __init__( branch=None, no_pull=False, ): - self.subworkflow_name = subworkflow_name - self.no_prompts = no_prompts - self.pytest_args = pytest_args - - super().__init__("subworkflows", ".", remote_url, branch, no_pull) - - def run(self): - """Run test steps""" - if not self.no_prompts: - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" - ) - self._check_inputs() - self._set_profile() - self._check_profile() - self._run_pytests() - - def _check_inputs(self): - """Do more complex checks about supplied flags.""" - # Retrieving installed subworkflows - if self.repo_type == "modules": - installed_subwf = self.get_components_clone_modules() - else: - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - installed_subwf = modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - - # Get the subworkflow name if not specified - if self.subworkflow_name is None: - if self.no_prompts: - raise UserWarning( - "Subworkflow name not provided and prompts deactivated. Please provide the Subworkflow name SUBWORKFLOW." - ) - if not installed_subwf: - raise UserWarning( - f"No installed subworkflows were found from '{self.modules_repo.remote_url}'.\n" - f"Are you running the tests inside the nf-core/modules main directory?\n" - f"Otherwise, make sure that the directory structure is subworkflows/SUBWORKFLOW/ and tests/subworkflows/SUBWORKFLOW/" - ) - self.subworkflow_name = questionary.autocomplete( - "Subworkflow name:", - choices=installed_subwf, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Sanity check that the module directory exists - self._validate_folder_structure() - - def _validate_folder_structure(self): - """Validate that the modules follow the correct folder structure to run the tests: - - subworkflows/nf-core/SUBWORKFLOW/ - - tests/subworkflows/nf-core/SUBWORKFLOW/ - - """ - subworkflow_path = Path(self.default_subworkflows_path) / self.subworkflow_name - test_path = Path(self.default_subworkflows_tests_path) / self.subworkflow_name - - if not (self.dir / subworkflow_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{subworkflow_path}'. Should be SUBWORKFLOW. Are you running the tests inside the nf-core/modules main directory?" - ) - if not (self.dir / test_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{test_path}'. Should be SUBWORKFLOW. " - "Are you running the tests inside the nf-core/modules main directory? " - "Do you have tests for the specified module?" - ) - - def _set_profile(self): - """Set $PROFILE env variable. - The config expects $PROFILE and Nextflow fails if it's not set. - """ - if os.environ.get("PROFILE") is None: - os.environ["PROFILE"] = "" - if self.no_prompts: - log.info( - "Setting environment variable '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " - "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." - ) - else: - question = { - "type": "list", - "name": "profile", - "message": "Choose software profile", - "choices": ["Docker", "Singularity", "Conda"], - } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() - os.environ["PROFILE"] = profile - log.info(f"Setting environment variable '$PROFILE' to '{profile}'") - - def _check_profile(self): - """Check if profile is available""" - profile = os.environ.get("PROFILE") - # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "conda"] - if profile in valid_nextflow_profiles: - if not which(profile): - raise UserWarning(f"Command '{profile}' not found - is it installed?") - else: - raise UserWarning( - f"The PROFILE '{profile}' set in the shell environment is not valid.\n" - f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." - ) - - def _run_pytests(self): - """Given a subworkflow name, run tests.""" - # Print nice divider line - console = rich.console.Console() - console.rule(self.subworkflow_name, style="black") - - # Set pytest arguments - command_args = ["--tag", f"{self.subworkflow_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] - command_args += self.pytest_args - - # Run pytest - log.info(f"Running pytest for module '{self.subworkflow_name}'") - sys.exit(pytest.main(command_args)) + super().__init__( + component_type="subworkflows", + module_name=subworkflow_name, # TODO: modify when components_test.py is refactored to work with subworkflows + no_prompts=no_prompts, + pytest_args=pytest_args, + remote_url=remote_url, + branch=branch, + no_pull=no_pull, + ) From 6cb550fe494a43657a54734b8c4c926b6e8eb403 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 14:40:49 +0100 Subject: [PATCH 490/854] adapt components test for modules and subworkflows --- nf_core/components/components_test.py | 69 ++++++++++++++--------- nf_core/subworkflows/subworkflows_test.py | 4 +- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 69218e5e5d..048c1a8f83 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -18,13 +18,13 @@ class ComponentsTest(ComponentCommand): """ - Class to run module pytests. + Class to run module and subworkflow pytests. ... Attributes ---------- - module_name : str + component_name : str name of the tool to run tests for no_prompts : bool flat indicating if prompts are used @@ -36,7 +36,7 @@ class ComponentsTest(ComponentCommand): run(): Run test steps _check_inputs(): - Check inputs. Ask for module_name if not provided and check that the directory exists + Check inputs. Ask for component_name if not provided and check that the directory exists _set_profile(): Set software profile _run_pytests(self): @@ -46,7 +46,7 @@ class ComponentsTest(ComponentCommand): def __init__( self, component_type, - module_name=None, + component_name=None, no_prompts=False, pytest_args="", remote_url=None, @@ -54,7 +54,7 @@ def __init__( no_pull=False, ): super().__init__(component_type=component_type, dir=".", remote_url=remote_url, branch=branch, no_pull=no_pull) - self.module_name = module_name + self.component_name = component_name self.no_prompts = no_prompts self.pytest_args = pytest_args @@ -76,27 +76,34 @@ def _check_inputs(self): # Retrieving installed modules if self.repo_type == "modules": - installed_modules = self.get_components_clone_modules() + installed_components = self.get_components_clone_modules() else: modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() - installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url) + if self.component_type == "modules": + installed_components = modules_json.get_all_modules().get(self.modules_repo.remote_url) + elif self.component_type == "subworkflows": + modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - # Get the tool name if not specified - if self.module_name is None: + # Get the component name if not specified + if self.component_name is None: if self.no_prompts: raise UserWarning( - "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." + f"{self.component_type[:-1].title()} name not provided and prompts deactivated. Please provide the {self.component_type[:-1]} name{' as TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}." ) - if not installed_modules: + if not installed_components: + if self.component_type == "modules": + dir_structure_message = f"modules/{self.modules_repo.repo_path}/TOOL/SUBTOOL/ and tests/modules/{self.modules_repo.repo_path}/TOOLS/SUBTOOL/" + elif self.component_type == "subworkflows": + dir_structure_message = f"subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/ and tests/subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/" raise UserWarning( - f"No installed modules were found from '{self.modules_repo.remote_url}'.\n" - f"Are you running the tests inside the nf-core/modules main directory?\n" - f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/" + f"No installed {self.component_type} were found from '{self.modules_repo.remote_url}'.\n" + f"Are you running the tests inside the repository root directory?\n" + f"Make sure that the directory structure is {dir_structure_message}" ) - self.module_name = questionary.autocomplete( + self.component_name = questionary.autocomplete( "Tool name:", - choices=installed_modules, + choices=installed_components, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -107,19 +114,25 @@ def _validate_folder_structure(self): """Validate that the modules follow the correct folder structure to run the tests: - modules/nf-core/TOOL/SUBTOOL/ - tests/modules/nf-core/TOOL/SUBTOOL/ - + or + - subworkflows/nf-core/SUBWORKFLOW/ + - tests/subworkflows/nf-core/SUBWORKFLOW/ """ - module_path = Path(self.default_modules_path) / self.module_name - test_path = Path(self.default_tests_path) / self.module_name - - if not (self.dir / module_path).is_dir(): + if self.component_type == "modules": + component_path = Path(self.default_modules_path) / self.component_name + test_path = Path(self.default_tests_path) / self.component_name + elif self.component_type == "subworkflows": + component_path = Path(self.default_subworkflows_path) / self.component_name + test_path = Path(self.default_subworkflows_tests_path) / self.component_name + + if not (self.dir / component_path).is_dir(): raise UserWarning( - f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" + f"Cannot find directory '{component_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. Are you running the tests inside the modules repository root directory?" ) if not (self.dir / test_path).is_dir(): raise UserWarning( - f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. " - "Are you running the tests inside the nf-core/modules main directory? " + f"Cannot find directory '{test_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. " + "Are you running the tests inside the modules repository root directory? " "Do you have tests for the specified module?" ) @@ -162,15 +175,15 @@ def _check_profile(self): ) def _run_pytests(self): - """Given a module name, run tests.""" + """Given a module/subworkflow name, run tests.""" # Print nice divider line console = rich.console.Console() - console.rule(self.module_name, style="black") + console.rule(self.component_name, style="black") # Set pytest arguments - command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] + command_args = ["--tag", f"{self.component_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] command_args += self.pytest_args # Run pytest - log.info(f"Running pytest for module '{self.module_name}'") + log.info(f"Running pytest for {self.component_type[:-1]} '{self.component_name}'") sys.exit(pytest.main(command_args)) diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 803e808bde..1f29418f49 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -13,7 +13,7 @@ class SubworkflowsTest(ComponentsTest): def __init__( self, - subworkflow_name=None, + component_name=None, no_prompts=False, pytest_args="", remote_url=None, @@ -22,7 +22,7 @@ def __init__( ): super().__init__( component_type="subworkflows", - module_name=subworkflow_name, # TODO: modify when components_test.py is refactored to work with subworkflows + module_name=component_name, no_prompts=no_prompts, pytest_args=pytest_args, remote_url=remote_url, From 5d598f202f911e3cc96fd2447b24e56995328688 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 8 Nov 2022 14:57:01 +0100 Subject: [PATCH 491/854] add component_name argument and fix text in tests --- nf_core/modules/modules_test.py | 2 +- nf_core/subworkflows/subworkflows_test.py | 4 ++-- tests/modules/modules_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py index f02ea11cdb..94dfd344c6 100644 --- a/nf_core/modules/modules_test.py +++ b/nf_core/modules/modules_test.py @@ -22,7 +22,7 @@ def __init__( ): super().__init__( component_type="modules", - module_name=module_name, + component_name=module_name, no_prompts=no_prompts, pytest_args=pytest_args, remote_url=remote_url, diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py index 1f29418f49..d072ff678a 100644 --- a/nf_core/subworkflows/subworkflows_test.py +++ b/nf_core/subworkflows/subworkflows_test.py @@ -13,7 +13,7 @@ class SubworkflowsTest(ComponentsTest): def __init__( self, - component_name=None, + subworkflow_name=None, no_prompts=False, pytest_args="", remote_url=None, @@ -22,7 +22,7 @@ def __init__( ): super().__init__( component_type="subworkflows", - module_name=component_name, + component_name=subworkflow_name, no_prompts=no_prompts, pytest_args=pytest_args, remote_url=remote_url, diff --git a/tests/modules/modules_test.py b/tests/modules/modules_test.py index ce57312bf1..eb207fa28b 100644 --- a/tests/modules/modules_test.py +++ b/tests/modules/modules_test.py @@ -25,7 +25,7 @@ def test_modules_test_no_name_no_prompts(self): meta_builder = nf_core.modules.ModulesTest(None, True, "") with pytest.raises(UserWarning) as excinfo: meta_builder._check_inputs() - assert "Tool name not provided and prompts deactivated." in str(excinfo.value) + assert "Module name not provided and prompts deactivated." in str(excinfo.value) def test_modules_test_no_installed_modules(self): From e4bd1be82ad60e6478d2c225458ea093d3928eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 8 Nov 2022 16:47:50 +0100 Subject: [PATCH 492/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/modules/modules_json.py | 2 +- tests/modules/modules_json.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 9fc0275163..cc52ee896c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -228,7 +228,7 @@ def dir_tree_uncovered(self, components_directory, repos): def determine_branches_and_shas(self, component_type, install_dir, remote_url, components): """ - Determines what branch and commit sha each module/subworkflow in the pipeline belong to + Determines what branch and commit sha each module/subworkflow in the pipeline belongs to Assumes all modules/subworkflows are installed from the default branch. If it fails to find the module/subworkflow in the default branch, it prompts the user with the available branches diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 8ad7c77b5c..20eee54e30 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -60,7 +60,6 @@ def test_mod_json_create(self): # Get the contents of the file mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() - print(mod_json) mods = ["fastqc", "multiqc"] for mod in mods: From 6eaef489f0885f6336d664f791e9aa39bf24c27b Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 9 Nov 2022 13:41:37 +0000 Subject: [PATCH 493/854] Fix variable name --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index bbe629adfc..7fd049f6fc 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -87,7 +87,7 @@ def prompt_component_version_sha(component_name, component_type, modules_repo, i git_sha = "" page_nbr = 1 - all_commits = modules_repo.get_component_git_log(component_name, subworkflow) + all_commits = modules_repo.get_component_git_log(component_name, component_type) next_page_commits = [next(all_commits, None) for _ in range(10)] next_page_commits = [commit for commit in next_page_commits if commit is not None] From 33838b01fe50b1121965abccc9dd8dca81fa7b9f Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 9 Nov 2022 18:18:22 +0100 Subject: [PATCH 494/854] first running version of sw remove, test are still failing --- nf_core/__main__.py | 28 ++++++++++ nf_core/components/components_install.py | 5 +- nf_core/components/components_test.py | 2 +- nf_core/components/remove.py | 68 ++++++++++++++++++++++++ nf_core/modules/modules_json.py | 20 +++++-- nf_core/modules/remove.py | 59 +------------------- nf_core/subworkflows/__init__.py | 1 + nf_core/subworkflows/remove.py | 10 ++++ tests/subworkflows/list.py | 4 +- tests/subworkflows/remove.py | 40 ++++++++++++++ tests/test_subworkflows.py | 12 +++-- 11 files changed, 179 insertions(+), 70 deletions(-) create mode 100644 nf_core/components/remove.py create mode 100644 nf_core/subworkflows/remove.py create mode 100644 tests/subworkflows/remove.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 86ec88e1ee..ab906bd658 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1063,6 +1063,34 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin sys.exit(1) +# nf-core subworkflows remove +@subworkflows.command() +@click.pass_context +@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", +) +def remove(ctx, dir, subworkflow): + """ + Remove a subworkflow from a pipeline. + """ + try: + module_remove = nf_core.subworkflows.SubworkflowRemove( + dir, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + module_remove.remove(subworkflow) + except (UserWarning, LookupError) as e: + log.critical(e) + sys.exit(1) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py index 5ad00695bc..806c0ed57e 100644 --- a/nf_core/components/components_install.py +++ b/nf_core/components/components_install.py @@ -99,5 +99,8 @@ def clean_modules_json(component, component_type, modules_repo, modules_json): log.info( f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" ) - modules_json.remove_entry(component_type, component, repo_to_remove, modules_repo.repo_path) + removed_by = component if component_type == "subworkflows" else None + modules_json.remove_entry( + component_type, component, repo_to_remove, modules_repo.repo_path, removed_by=removed_by + ) return component_values["installed_by"] diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 048c1a8f83..6a2cba5001 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -102,7 +102,7 @@ def _check_inputs(self): f"Make sure that the directory structure is {dir_structure_message}" ) self.component_name = questionary.autocomplete( - "Tool name:", + f"{self.component_type[:-1]} name:", choices=installed_components, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py new file mode 100644 index 0000000000..74476aeefc --- /dev/null +++ b/nf_core/components/remove.py @@ -0,0 +1,68 @@ +import logging +from pathlib import Path + +import questionary + +import nf_core.utils +from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson + +log = logging.getLogger(__name__) + + +class ComponentRemove(ComponentCommand): + def __init__(self, component_type, pipeline_dir): + super().__init__(component_type, pipeline_dir) + + def remove(self, component): + """ + Remove an already installed module/subworkflow + This command only works for modules/subworkflows that are installed from 'nf-core/modules' + """ + if self.repo_type == "modules": + log.error(f"You cannot remove a {self.component_type[:-1]} in a clone of nf-core/modules") + return False + + # Check modules directory structure + self.check_modules_structure() + + # Check whether pipeline is valid and with a modules.json file + self.has_valid_directory() + self.has_modules_file() + + repo_dir = self.modules_repo.fullname + repo_path = self.modules_repo.repo_path + if component is None: + component = questionary.autocomplete( + f"{self.component_type[:-1]} name:", + choices=self.components_from_repo(repo_dir), + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Get the module/subworkflow directory + module_dir = Path(self.dir, self.component_type, repo_path, component) + + # Load the modules.json file + modules_json = ModulesJson(self.dir) + modules_json.load() + + # Verify that the module/subworkflow is actually installed + if not module_dir.exists(): + log.error(f"Module directory does not exist: '{module_dir}'") + + if modules_json.module_present(component, self.modules_repo.remote_url, repo_path): + log.error(f"Found entry for '{component}' in 'modules.json'. Removing...") + modules_json.remove_entry(self.component_type, component, self.modules_repo.remote_url, repo_path) + return False + + # Remove entry from modules.json + removed_by = component if self.component_type == "subworkflows" else None + removed = False + removed = modules_json.remove_entry( + self.component_type, component, self.modules_repo.remote_url, repo_path, removed_by=removed_by + ) + # Remove the module files + if removed: + removed = self.clear_component_dir(component, module_dir) + + return removed diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cc52ee896c..25e52b36ca 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -670,7 +670,7 @@ def update_subworkflow( if write_file: self.dump() - def remove_entry(self, component_type, name, repo_url, install_dir): + def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=None): """ Removes an entry from the 'modules.json' file. @@ -680,14 +680,26 @@ def remove_entry(self, component_type, name, repo_url, install_dir): repo_url (str): URL of the repository containing the module install_dir (str): Name of the directory where modules are installed Returns: - (bool): True if the removal was successful, False otherwise + (bool): True if the entry has actually been removed from the modules.json, False otherwise """ + + if removed_by is None: + removed_by = component_type if not self.modules_json: return False if repo_url in self.modules_json.get("repos", {}): repo_entry = self.modules_json["repos"][repo_url] if name in repo_entry[component_type].get(install_dir, {}): - repo_entry[component_type][install_dir].pop(name) + if removed_by in repo_entry[component_type][install_dir][name]["installed_by"]: + repo_entry[component_type][install_dir][name]["installed_by"].remove(removed_by) + if len(repo_entry[component_type][install_dir][name]["installed_by"]) == 0: + repo_entry[component_type][install_dir].pop(name) + return True + else: + log.error( + f"Could not find 'installed_by' entry for '{removed_by}' in 'modules.json' file. Did you install it first?" + ) + else: log.warning( f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file." @@ -700,7 +712,7 @@ def remove_entry(self, component_type, name, repo_url, install_dir): return False self.dump() - return True + return False def add_patch_entry(self, module_name, repo_url, install_dir, patch_filename, write_file=True): """ diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 1930ecb879..72c94927a0 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -1,65 +1,10 @@ import logging -from pathlib import Path -import questionary - -import nf_core.utils -from nf_core.components.components_command import ComponentCommand - -from .modules_json import ModulesJson +from nf_core.components.remove import ComponentRemove log = logging.getLogger(__name__) -class ModuleRemove(ComponentCommand): +class ModuleRemove(ComponentRemove): def __init__(self, pipeline_dir): super().__init__("modules", pipeline_dir) - - def remove(self, module): - """ - Remove an already installed module - This command only works for modules that are installed from 'nf-core/modules' - """ - if self.repo_type == "modules": - log.error("You cannot remove a module in a clone of nf-core/modules") - return False - - # Check modules directory structure - self.check_modules_structure() - - # Check whether pipeline is valid and with a modules.json file - self.has_valid_directory() - self.has_modules_file() - - repo_dir = self.modules_repo.fullname - repo_path = self.modules_repo.repo_path - if module is None: - module = questionary.autocomplete( - "Tool name:", - choices=self.components_from_repo(repo_dir), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Get the module directory - module_dir = Path(self.dir, "modules", repo_path, module) - - # Load the modules.json file - modules_json = ModulesJson(self.dir) - modules_json.load() - - # Verify that the module is actually installed - if not module_dir.exists(): - log.error(f"Module directory does not exist: '{module_dir}'") - - if modules_json.module_present(module, self.modules_repo.remote_url, repo_path): - log.error(f"Found entry for '{module}' in 'modules.json'. Removing...") - modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path) - return False - - log.info(f"Removing {module}") - - # Remove entry from modules.json - modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path) - - # Remove the module - return self.clear_component_dir(module, module_dir) diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index f10c850d8d..ebb2e3821e 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,5 +1,6 @@ from .create import SubworkflowCreate from .install import SubworkflowInstall from .list import SubworkflowList +from .remove import SubworkflowsRemove from .subworkflows_test import SubworkflowsTest from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/remove.py b/nf_core/subworkflows/remove.py new file mode 100644 index 0000000000..bd538a4e5a --- /dev/null +++ b/nf_core/subworkflows/remove.py @@ -0,0 +1,10 @@ +import logging + +from nf_core.components.remove import ComponentRemove + +log = logging.getLogger(__name__) + + +class SubworkflowsRemove(ComponentRemove): + def __init__(self, pipeline_dir): + super().__init__("subworkflows", pipeline_dir) diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index 8daf5fb599..c65999d42c 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -29,7 +29,7 @@ def test_subworkflows_list_remote_gitlab(self): def test_subworkflows_install_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - self.sw_install.install("bam_sort_stats_samtools") + self.subworkflow_install.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() console = Console(record=True) @@ -40,7 +40,7 @@ def test_subworkflows_install_and_list_subworkflows(self): def test_subworkflows_install_gitlab_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - self.sw_install_gitlab.install("bam_sort_stats_samtools") + self.subworkflow_install_gitlab.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() console = Console(record=True) diff --git a/tests/subworkflows/remove.py b/tests/subworkflows/remove.py new file mode 100644 index 0000000000..28100c5e52 --- /dev/null +++ b/tests/subworkflows/remove.py @@ -0,0 +1,40 @@ +from pathlib import Path + + +def test_subworkflows_remove_uninstalled_subworkflow(self): + """Test removing subworkflow without installing it""" + assert self.subworkflow_remove.remove("bam_sort_stats_samtools") is False + + +def test_subworkflows_remove_subworkflow(self): + """Test removing subworkflow and all it's dependencies after installing it""" + self.subworkflow_install.install("bam_sort_stats_samtools") + + subworkflow_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "subworkflows") + bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools") + bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") + samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "modules", "samtools", "index") + + assert self.subworkflow_remove.remove("bam_sort_stats_samtools") + assert Path.exists(subworkflow_path) is False + assert Path.exists(bam_sort_stats_samtools_path) is False + assert Path.exists(bam_stats_samtools_path) is False + assert Path.exists(samtools_index_path) is False + + +def test_subworkflows_remove_one_of_two_subworkflow(self): + """Test removing subworkflow and all it's dependencies after installing it""" + self.subworkflow_install.install("bam_sort_stats_samtools") + self.subworkflow_install.install("bam_stats_samtools") + + subworkflow_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "subworkflows") + bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools") + bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") + samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "modules", "samtools", "index") + + assert self.subworkflow_remove.remove("bam_sort_stats_samtools") + + assert Path.exists(subworkflow_path) is False + assert Path.exists(bam_sort_stats_samtools_path) is False + assert Path.exists(bam_stats_samtools_path) is True + assert Path.exists(samtools_index_path) is True diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index be15e2f15b..59e508111d 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -61,11 +61,8 @@ def setUp(self): # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) - # Set up install objects - self.sw_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=False) - self.sw_install_gitlab = nf_core.subworkflows.SubworkflowInstall( - self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH - ) + # Set up remove objects + self.subworkflow_remove = nf_core.subworkflows.SubworkflowsRemove(self.pipeline_dir) ############################################ # Test of the individual modules commands. # @@ -91,6 +88,11 @@ def setUp(self): test_subworkflows_list_remote, test_subworkflows_list_remote_gitlab, ) + from .subworkflows.remove import ( + test_subworkflows_remove_one_of_two_subworkflow, + test_subworkflows_remove_subworkflow, + test_subworkflows_remove_uninstalled_subworkflow, + ) from .subworkflows.subworkflows_test import ( test_subworkflows_test_check_inputs, test_subworkflows_test_no_installed_subworkflows, From acd1605360e83684c620c5c193913d263033c6c8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 9 Nov 2022 22:42:22 +0100 Subject: [PATCH 495/854] Nextflow syntax highlighting for nf-test I think that this should give the nf-test files syntax highlighting on Github.com --- nf_core/pipeline-template/.gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.gitattributes b/nf_core/pipeline-template/.gitattributes index 050bb12035..7a2dabc293 100644 --- a/nf_core/pipeline-template/.gitattributes +++ b/nf_core/pipeline-template/.gitattributes @@ -1,3 +1,4 @@ *.config linguist-language=nextflow +*.nf.test linguist-language=nextflow modules/nf-core/** linguist-generated subworkflows/nf-core/** linguist-generated From 8cab671cb603ec75585d3d05b3d3e6a8dea8271d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 10 Nov 2022 10:18:30 +0100 Subject: [PATCH 496/854] Don't print the filename + line number in log calls --- nf_core/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 86ec88e1ee..e48b347295 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -135,6 +135,7 @@ def nf_core_cli(verbose, log_file): level=logging.DEBUG if verbose else logging.INFO, console=rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()), show_time=False, + show_path=verbose, # True if verbose, false otherwise markup=True, ) ) From 551acbfc482bab532380f64184816a65779718a0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 10 Nov 2022 10:19:54 +0100 Subject: [PATCH 497/854] Update changelog [skip-ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbbc7c0ab8..a4dd46decb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ - Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001)) - Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999)) - Substitute ModulesCommand and SubworkflowsCommand by ComponentsCommand ([#2000](https://github.com/nf-core/tools/pull/2000)) +- Don't print source file + line number on logging messages (except when verbose) ([#2015](https://github.com/nf-core/tools/pull/2015)) ### Modules From c81480a76af773c82242d142e2a6ba7e3a48e660 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 10 Nov 2022 10:29:54 +0100 Subject: [PATCH 498/854] Refactor --hide-progress to be at the top level CLI --- .github/workflows/create-lint-wf.yml | 4 ++-- CHANGELOG.md | 1 + nf_core/__main__.py | 36 +++++++++++----------------- nf_core/modules/lint/__init__.py | 4 +++- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 4dc87c0a14..4e19bc65d2 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -86,7 +86,7 @@ jobs: # Run nf-core linting - name: nf-core lint - run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --fail-warned + run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned # Run the other nf-core commands - name: nf-core list @@ -102,7 +102,7 @@ jobs: run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1 - name: nf-core lint in release mode - run: nf-core --log-file log.txt lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release + run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release - name: nf-core modules install run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force diff --git a/CHANGELOG.md b/CHANGELOG.md index fbbc7c0ab8..68cb63b347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### General +- Refactor CLI flag `--hide-progress` to be at the top-level group, like `--verbose` ([#2016](https://github.com/nf-core/tools/pull/2016)) - Fix error in tagging GitPod docker images during releases - `nf-core sync` now supports the template YAML file using `-t/--template-yaml`. - Fix bug when updating modules from old version in old folder structure diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 86ec88e1ee..771fe0de72 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -104,23 +104,13 @@ def run_nf_core(): nf_core_cli(auto_envvar_prefix="NFCORE") -# taken from https://github.com/pallets/click/issues/108#issuecomment-194465429 -_common_options = [ - click.option("--hide-progress", is_flag=True, default=False, help="Don't show progress bars."), -] - - -def common_options(func): - for option in reversed(_common_options): - func = option(func) - return func - - @click.group(context_settings=dict(help_option_names=["-h", "--help"])) @click.version_option(nf_core.__version__) @click.option("-v", "--verbose", is_flag=True, default=False, help="Print verbose output to the console.") +@click.option("--hide-progress", is_flag=True, default=False, help="Don't show progress bars.") @click.option("-l", "--log-file", help="Save a verbose log to a file.", metavar="") -def nf_core_cli(verbose, log_file): +@click.pass_context +def nf_core_cli(ctx, verbose, hide_progress, log_file): """ nf-core/tools provides a set of helper tools for use with nf-core Nextflow pipelines. @@ -146,6 +136,11 @@ def nf_core_cli(verbose, log_file): log_fh.setFormatter(logging.Formatter("[%(asctime)s] %(name)-20s [%(levelname)-7s] %(message)s")) log.addHandler(log_fh) + ctx.obj = { + "verbose": verbose, + "hide_progress": hide_progress or verbose, # Always hide progress bar with verbose logging + } + # nf-core list @nf_core_cli.command() @@ -328,8 +323,8 @@ def create(name, description, author, version, no_git, force, outdir, template_y @click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures") @click.option("--markdown", type=str, metavar="", help="File to write linting results to (Markdown)") @click.option("--json", type=str, metavar="", help="File to write linting results to (JSON)") -@common_options -def lint(dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, hide_progress): +@click.pass_context +def lint(ctx, dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json): """ Check pipeline code against nf-core guidelines. @@ -351,7 +346,7 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdow # Run the lint tests! try: lint_obj, module_lint_obj = nf_core.lint.run_linting( - dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, hide_progress + dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, ctx.obj["hide_progress"] ) if len(lint_obj.failed) + len(module_lint_obj.failed) > 0: sys.exit(1) @@ -729,10 +724,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.option("--local", is_flag=True, help="Run additional lint tests for local modules") @click.option("--passed", is_flag=True, help="Show passed tests") @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") -@common_options -def lint( - ctx, tool, dir, key, all, fail_warned, local, passed, fix_version, hide_progress -): # pylint: disable=redefined-outer-name +def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version): # pylint: disable=redefined-outer-name """ Lint one or more modules in a directory. @@ -749,13 +741,13 @@ def lint( ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], - hide_progress, + ctx.obj["hide_progress"], ) module_lint.lint( module=tool, key=key, all_modules=all, - hide_progress=hide_progress, + hide_progress=ctx.obj["hide_progress"], print_results=True, local=local, show_passed=passed, diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index dff57f9da5..10063f90c5 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -72,7 +72,9 @@ def __init__( no_pull=False, hide_progress=False, ): - super().__init__("modules", dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=False) + super().__init__( + "modules", dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=hide_progress + ) self.fail_warned = fail_warned self.passed = [] From b222d899db2560aa59ade89b0e1c2551cef236de Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 10 Nov 2022 11:32:15 +0100 Subject: [PATCH 499/854] Remove CLI param from pytest --- tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index d8d9eb29c7..f991681b0f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -273,7 +273,6 @@ def test_lint(self, mock_lint, mock_is_pipeline): "fail-warned": None, "markdown": "output_file.md", "json": "output_file.json", - "hide-progress": None, } cmd = ["lint"] + self.assemble_params(params) From 6104d1c9fc816ebe1db82fe9f84b3b65700db573 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 10 Nov 2022 14:52:05 +0100 Subject: [PATCH 500/854] don't write file when applying a patch during update --- nf_core/modules/update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 297e893401..fbc6c55949 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -173,7 +173,7 @@ def update(self, module=None): if patch_relpath is not None: patch_successful = self.try_apply_patch( - module, modules_repo.repo_path, patch_relpath, module_dir, module_install_dir + module, modules_repo.repo_path, patch_relpath, module_dir, module_install_dir, write_file=False ) if patch_successful: log.info(f"Module '{module_fullname}' patched successfully") @@ -630,7 +630,7 @@ def move_files_from_tmp_dir(self, module, install_folder, repo_path, new_version log.info(f"Updating '{repo_path}/{module}'") log.debug(f"Updating module '{module}' to {new_version} from {repo_path}") - def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_install_dir): + def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_install_dir, write_file=True): """ Try applying a patch file to the new module files @@ -698,7 +698,7 @@ def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_i # Add the patch file to the modules.json file self.modules_json.add_patch_entry( - module, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=True + module, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=write_file ) return True From f0fed33eae441f2e4520914e78188641d7a8a987 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 10 Nov 2022 14:53:10 +0100 Subject: [PATCH 501/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4dd46decb..c01115a34b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ ### Modules - Update patch file paths if the modules directory has the old structure ([#1878](https://github.com/nf-core/tools/pull/1878)) +- Don't write to `modules.json` file when applying a patch file during `nf-core modules update` ### Subworkflows From 3467f5833062f2c6da28aafc01b78c4f8903884a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 10 Nov 2022 17:08:08 +0100 Subject: [PATCH 502/854] first refactoring of update commands --- nf_core/components/update.py | 704 ++++++++++++++++++++++++++++++++ nf_core/modules/modules_json.py | 23 ++ nf_core/modules/update.py | 702 +------------------------------ nf_core/subworkflows/update.py | 30 ++ 4 files changed, 771 insertions(+), 688 deletions(-) create mode 100644 nf_core/components/update.py create mode 100644 nf_core/subworkflows/update.py diff --git a/nf_core/components/update.py b/nf_core/components/update.py new file mode 100644 index 0000000000..7ba6cff81e --- /dev/null +++ b/nf_core/components/update.py @@ -0,0 +1,704 @@ +import logging +import os +import shutil +import tempfile +from pathlib import Path + +import questionary + +import nf_core.modules.modules_utils +import nf_core.utils +from nf_core.components.components_command import ComponentCommand +from nf_core.components.components_utils import prompt_component_version_sha +from nf_core.modules.modules_differ import ModulesDiffer +from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import ModulesRepo +from nf_core.utils import plural_es, plural_s, plural_y + +log = logging.getLogger(__name__) + + +class ComponentUpdate(ComponentCommand): + def __init__( + self, + pipeline_dir, + component_type, + force=False, + prompt=False, + sha=None, + update_all=False, + show_diff=None, + save_diff_fn=None, + remote_url=None, + branch=None, + no_pull=False, + ): + super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.force = force + self.prompt = prompt + self.sha = sha + self.update_all = update_all + self.show_diff = show_diff + self.save_diff_fn = save_diff_fn + self.module = None + self.update_config = None + self.modules_json = ModulesJson(self.dir) + self.branch = branch + + def _parameter_checks(self): + """Checks the compatibilty of the supplied parameters. + + Raises: + UserWarning: if any checks fail. + """ + + if self.save_diff_fn and self.show_diff: + raise UserWarning("Either `--preview` or `--save_diff` can be specified, not both.") + + if self.update_all and self.module: + raise UserWarning("Either a module or the '--all' flag can be specified, not both.") + + if self.repo_type == "modules": + raise UserWarning("Modules in clones of nf-core/modules can not be updated.") + + if self.prompt and self.sha is not None: + raise UserWarning("Cannot use '--sha' and '--prompt' at the same time.") + + if not self.has_valid_directory(): + raise UserWarning("The command was not run in a valid pipeline directory.") + + def update(self, module=None): + """Updates a specified module or all modules modules in a pipeline. + + Args: + module (str): The name of the module to update. + + Returns: + bool: True if the update was successful, False otherwise. + """ + self.module = module + + tool_config = nf_core.utils.load_tools_config(self.dir) + self.update_config = tool_config.get("update", {}) + + self._parameter_checks() + + # Check modules directory structure + self.check_modules_structure() + + # Verify that 'modules.json' is consistent with the installed modules + self.modules_json.check_up_to_date() + + if not self.update_all and module is None: + choices = ["All modules", "Named module"] + self.update_all = ( + questionary.select( + "Update all modules or a single named module?", + choices=choices, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + == "All modules" + ) + + # Verify that the provided SHA exists in the repo + if self.sha is not None and not self.modules_repo.sha_exists_on_branch(self.sha): + log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") + return False + + # Get the list of modules to update, and their version information + modules_info = self.get_all_modules_info() if self.update_all else [self.get_single_module_info(module)] + + # Save the current state of the modules.json + old_modules_json = self.modules_json.get_modules_json() + + # Ask if we should show the diffs (unless a filename was already given on the command line) + if not self.save_diff_fn and self.show_diff is None: + diff_type = questionary.select( + "Do you want to view diffs of the proposed changes?", + choices=[ + {"name": "No previews, just update everything", "value": 0}, + {"name": "Preview diff in terminal, choose whether to update files", "value": 1}, + {"name": "Just write diffs to a patch file", "value": 2}, + ], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + self.show_diff = diff_type == 1 + self.save_diff_fn = diff_type == 2 + + if self.save_diff_fn: # True or a string + self.setup_diff_file() + + # Loop through all modules to be updated + # and do the requested action on them + exit_value = True + all_patches_successful = True + for modules_repo, module, sha, patch_relpath in modules_info: + module_fullname = str(Path("modules", modules_repo.repo_path, module)) + # Are we updating the files in place or not? + dry_run = self.show_diff or self.save_diff_fn + + current_version = self.modules_json.get_module_version( + module, modules_repo.remote_url, modules_repo.repo_path + ) + + # Set the temporary installation folder + install_tmp_dir = Path(tempfile.mkdtemp()) + module_install_dir = install_tmp_dir / module + + # Compute the module directory + module_dir = os.path.join(self.dir, "modules", modules_repo.repo_path, module) + + if sha is not None: + version = sha + elif self.prompt: + version = prompt_component_version_sha( + module, "modules", modules_repo=modules_repo, installed_sha=current_version + ) + else: + version = modules_repo.get_latest_component_version(module, self.component_type) + + if current_version is not None and not self.force: + if current_version == version: + if self.sha or self.prompt: + log.info(f"'{module_fullname}' is already installed at {version}") + else: + log.info(f"'{module_fullname}' is already up to date") + continue + + # Download module files + if not self.install_component_files(module, version, modules_repo, install_tmp_dir): + exit_value = False + continue + + if patch_relpath is not None: + patch_successful = self.try_apply_patch( + module, modules_repo.repo_path, patch_relpath, module_dir, module_install_dir + ) + if patch_successful: + log.info(f"Module '{module_fullname}' patched successfully") + else: + log.warning(f"Failed to patch module '{module_fullname}'. Will proceed with unpatched files.") + all_patches_successful &= patch_successful + + if dry_run: + if patch_relpath is not None: + if patch_successful: + log.info("Current installation is compared against patched version in remote.") + else: + log.warning("Current installation is compared against unpatched version in remote.") + # Compute the diffs for the module + if self.save_diff_fn: + log.info(f"Writing diff file for module '{module_fullname}' to '{self.save_diff_fn}'") + ModulesDiffer.write_diff_file( + self.save_diff_fn, + module, + modules_repo.repo_path, + module_dir, + module_install_dir, + current_version, + version, + dsp_from_dir=module_dir, + dsp_to_dir=module_dir, + ) + + elif self.show_diff: + ModulesDiffer.print_diff( + module, + modules_repo.repo_path, + module_dir, + module_install_dir, + current_version, + version, + dsp_from_dir=module_dir, + dsp_to_dir=module_dir, + ) + + # Ask the user if they want to install the module + dry_run = not questionary.confirm( + f"Update module '{module}'?", default=False, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + + if not dry_run: + # Clear the module directory and move the installed files there + self.move_files_from_tmp_dir(module, install_tmp_dir, modules_repo.repo_path, version) + # Update modules.json with newly installed module + self.modules_json.update(modules_repo, module, version, self.component_type) + else: + # Don't save to a file, just iteratively update the variable + self.modules_json.update(modules_repo, module, version, self.component_type, write_file=False) + + if self.save_diff_fn: + # Write the modules.json diff to the file + ModulesDiffer.append_modules_json_diff( + self.save_diff_fn, + old_modules_json, + self.modules_json.get_modules_json(), + Path(self.dir, "modules.json"), + ) + if exit_value: + log.info( + f"[bold magenta italic] TIP! [/] If you are happy with the changes in '{self.save_diff_fn}', you " + "can apply them by running the command :point_right:" + f" [bold magenta italic]git apply {self.save_diff_fn} [/]" + ) + elif not all_patches_successful: + log.info(f"Updates complete. Please apply failed patch{plural_es(modules_info)} manually") + else: + log.info("Updates complete :sparkles:") + + return exit_value + + def get_single_module_info(self, module): + """Collects the module repository, version and sha for a module. + + Information about the module version in the '.nf-core.yml' overrides + the '--sha' option + + Args: + module_name (str): The name of the module to get info for. + + Returns: + (ModulesRepo, str, str): The modules repo containing the module, + the module name, and the module version. + + Raises: + LookupError: If the module is not found either in the pipeline or the modules repo. + UserWarning: If the '.nf-core.yml' entry is not valid. + """ + # Check if there are any modules installed from the repo + repo_url = self.modules_repo.remote_url + modules = self.modules_json.get_all_modules().get(repo_url) + choices = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] + if repo_url not in self.modules_json.get_all_modules(): + raise LookupError(f"No modules installed from '{repo_url}'") + + if module is None: + module = questionary.autocomplete( + "Tool name:", + choices=choices, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Get module installation directory + install_dir = [dir for dir, m in modules if module == m][0] + + # Check if module is installed before trying to update + if module not in choices: + raise LookupError(f"Module '{module}' is not installed in pipeline and could therefore not be updated") + + # Check that the supplied name is an available module + if module and module not in self.modules_repo.get_avail_components(self.component_type): + raise LookupError( + f"Module '{module}' not found in list of available modules." + f"Use the command 'nf-core modules list remote' to view available software" + ) + + sha = self.sha + if module in self.update_config.get(self.modules_repo.remote_url, {}).get(install_dir, {}): + # If the module to update is in .nf-core.yml config file + config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(module) + if config_entry is not None and config_entry is not True: + if config_entry is False: + raise UserWarning("Module's update entry in '.nf-core.yml' is set to False") + if not isinstance(config_entry, str): + raise UserWarning("Module's update entry in '.nf-core.yml' is of wrong type") + + sha = config_entry + if self.sha is not None: + log.warning( + f"Found entry in '.nf-core.yml' for module '{module}' " + "which will override version specified with '--sha'" + ) + else: + log.info(f"Found entry in '.nf-core.yml' for module '{module}'") + log.info(f"Updating module to ({sha})") + + # Check if the update branch is the same as the installation branch + current_branch = self.modules_json.get_component_branch( + self.component_type, module, self.modules_repo.remote_url, install_dir + ) + new_branch = self.modules_repo.branch + if current_branch != new_branch: + log.warning( + f"You are trying to update the '{Path(install_dir, module)}' module from " + f"the '{new_branch}' branch. This module was installed from the '{current_branch}'" + ) + switch = questionary.confirm(f"Do you want to update using the '{current_branch}' instead?").unsafe_ask() + if switch: + # Change the branch + self.modules_repo.setup_branch(current_branch) + + # If there is a patch file, get its filename + patch_fn = self.modules_json.get_patch_fn(module, self.modules_repo.remote_url, install_dir) + + return (self.modules_repo, module, sha, patch_fn) + + def get_all_modules_info(self, branch=None): + """Collects the module repository, version and sha for all modules. + + Information about the module version in the '.nf-core.yml' overrides the '--sha' option. + + Returns: + [(ModulesRepo, str, str)]: A list of tuples containing a ModulesRepo object, + the module name, and the module version. + """ + if branch is not None: + use_branch = questionary.confirm( + "'--branch' was specified. Should this branch be used to update all modules?", default=False + ) + if not use_branch: + branch = None + skipped_repos = [] + skipped_modules = [] + overridden_repos = [] + overridden_modules = [] + modules_info = {} + # Loop through all the modules in the pipeline + # and check if they have an entry in the '.nf-core.yml' file + for repo_name, modules in self.modules_json.get_all_modules().items(): + if repo_name not in self.update_config or self.update_config[repo_name] is True: + # There aren't restrictions for the repository in .nf-core.yml file + modules_info[repo_name] = {} + for module_dir, module in modules: + try: + modules_info[repo_name][module_dir].append( + ( + module, + self.sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + self.sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ] + elif isinstance(self.update_config[repo_name], dict): + # If it is a dict, then there are entries for individual modules or module directories + for module_dir in set([dir for dir, _ in modules]): + if isinstance(self.update_config[repo_name][module_dir], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = self.update_config[repo_name][module_dir] + modules_info[repo_name] = {} + for dir, module in modules: + if module_dir == dir: + try: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ] + if self.sha is not None: + overridden_repos.append(repo_name) + elif self.update_config[repo_name][module_dir] is False: + for dir, module in modules: + if dir == module_dir: + skipped_modules.append(f"{module_dir}/{module}") + elif isinstance(self.update_config[repo_name][module_dir], dict): + # If it's a dict, there are entries for individual modules + dir_config = self.update_config[repo_name][module_dir] + modules_info[repo_name] = {} + for module_dir, module in modules: + if module not in dir_config or dir_config[module] is True: + try: + modules_info[repo_name][module_dir].append( + ( + module, + self.sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + self.sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ] + elif isinstance(dir_config[module], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = dir_config[module] + try: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ] + if self.sha is not None: + overridden_modules.append(module) + elif dir_config[module] is False: + # Otherwise the entry must be 'False' and we should ignore the module + skipped_modules.append(f"{module_dir}/{module}") + else: + raise UserWarning( + f"Module '{module}' in '{module_dir}' has an invalid entry in '.nf-core.yml'" + ) + elif isinstance(self.update_config[repo_name], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = self.update_config[repo_name] + modules_info[repo_name] = {} + for module_dir, module in modules: + try: + modules_info[repo_name][module_dir].append( + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ) + except KeyError: + modules_info[repo_name][module_dir] = [ + ( + module, + custom_sha, + self.modules_json.get_component_branch( + self.component_type, module, repo_name, module_dir + ), + ) + ] + if self.sha is not None: + overridden_repos.append(repo_name) + elif self.update_config[repo_name] is False: + skipped_repos.append(repo_name) + else: + raise UserWarning(f"Repo '{repo_name}' has an invalid entry in '.nf-core.yml'") + + if skipped_repos: + skipped_str = "', '".join(skipped_repos) + log.info(f"Skipping modules in repositor{plural_y(skipped_repos)}: '{skipped_str}'") + + if skipped_modules: + skipped_str = "', '".join(skipped_modules) + log.info(f"Skipping module{plural_s(skipped_modules)}: '{skipped_str}'") + + if overridden_repos: + overridden_str = "', '".join(overridden_repos) + log.info( + f"Overriding '--sha' flag for modules in repositor{plural_y(overridden_repos)} " + f"with '.nf-core.yml' entry: '{overridden_str}'" + ) + + if overridden_modules: + overridden_str = "', '".join(overridden_modules) + log.info( + f"Overriding '--sha' flag for module{plural_s(overridden_modules)} with " + f"'.nf-core.yml' entry: '{overridden_str}'" + ) + # Loop through modules_info and create on ModulesRepo object per remote and branch + repos_and_branches = {} + for repo_name, repo_content in modules_info.items(): + for module_dir, mods in repo_content.items(): + for mod, sha, mod_branch in mods: + if branch is not None: + mod_branch = branch + if (repo_name, mod_branch) not in repos_and_branches: + repos_and_branches[(repo_name, mod_branch)] = [] + repos_and_branches[(repo_name, mod_branch)].append((mod, sha)) + + # Create ModulesRepo objects + repo_objs_mods = [] + for (repo_url, branch), mods_shas in repos_and_branches.items(): + try: + modules_repo = ModulesRepo(remote_url=repo_url, branch=branch) + except LookupError as e: + log.warning(e) + log.info(f"Skipping modules in '{repo_url}'") + else: + repo_objs_mods.append((modules_repo, mods_shas)) + + # Flatten the list + modules_info = [(repo, mod, sha) for repo, mods_shas in repo_objs_mods for mod, sha in mods_shas] + + # Verify that that all modules and shas exist in their respective ModulesRepo, + # don't try to update those that don't + i = 0 + while i < len(modules_info): + repo, module, sha = modules_info[i] + if not repo.component_exists(module, self.component_type): + log.warning(f"Module '{module}' does not exist in '{repo.remote_url}'. Skipping...") + modules_info.pop(i) + elif sha is not None and not repo.sha_exists_on_branch(sha): + log.warning( + f"Git sha '{sha}' does not exists on the '{repo.branch}' of '{repo.remote_url}'. Skipping module '{module}'" + ) + modules_info.pop(i) + else: + i += 1 + + # Add patch filenames to the modules that have them + modules_info = [ + (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.remote_url, repo.repo_path)) + for repo, mod, sha in modules_info + ] + + return modules_info + + def setup_diff_file(self): + """Sets up the diff file. + + If the save diff option was chosen interactively, the user is asked to supply a name for the diff file. + + Then creates the file for saving the diff. + """ + if self.save_diff_fn is True: + # From questionary - no filename yet + self.save_diff_fn = questionary.path( + "Enter the filename: ", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + + self.save_diff_fn = Path(self.save_diff_fn) + + # Check if filename already exists (questionary or cli) + while self.save_diff_fn.exists(): + if questionary.confirm(f"'{self.save_diff_fn}' exists. Remove file?").unsafe_ask(): + os.remove(self.save_diff_fn) + break + self.save_diff_fn = questionary.path( + "Enter a new filename: ", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + self.save_diff_fn = Path(self.save_diff_fn) + + # This guarantees that the file exists after calling the function + self.save_diff_fn.touch() + + def move_files_from_tmp_dir(self, module, install_folder, repo_path, new_version): + """Move the files from the temporary to the installation directory. + + Args: + module (str): The module name. + install_folder [str]: The path to the temporary installation directory. + repo_path (str): The name of the directory where modules are installed + new_version (str): The version of the module that was installed. + """ + temp_module_dir = os.path.join(install_folder, module) + files = os.listdir(temp_module_dir) + pipeline_path = os.path.join(self.dir, "modules", repo_path, module) + + log.debug(f"Removing old version of module '{module}'") + self.clear_component_dir(module, pipeline_path) + + os.makedirs(pipeline_path) + for file in files: + path = os.path.join(temp_module_dir, file) + if os.path.exists(path): + shutil.move(path, os.path.join(pipeline_path, file)) + + log.info(f"Updating '{repo_path}/{module}'") + log.debug(f"Updating module '{module}' to {new_version} from {repo_path}") + + def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_install_dir): + """ + Try applying a patch file to the new module files + + + Args: + module (str): The name of the module + repo_path (str): The name of the repository where the module resides + patch_relpath (Path | str): The path to patch file in the pipeline + module_dir (Path | str): The module directory in the pipeline + module_install_dir (Path | str): The directory where the new module + file have been installed + + Returns: + (bool): Whether the patch application was successful + """ + module_fullname = str(Path(repo_path, module)) + log.info(f"Found patch for module '{module_fullname}'. Trying to apply it to new files") + + patch_path = Path(self.dir / patch_relpath) + module_relpath = Path("modules", repo_path, module) + + # Check that paths in patch file are updated + self.check_patch_paths(patch_path, module) + + # Copy the installed files to a new temporary directory to save them for later use + temp_dir = Path(tempfile.mkdtemp()) + temp_module_dir = temp_dir / module + shutil.copytree(module_install_dir, temp_module_dir) + + try: + new_files = ModulesDiffer.try_apply_patch(module, repo_path, patch_path, temp_module_dir) + except LookupError: + # Patch failed. Save the patch file by moving to the install dir + shutil.move(patch_path, Path(module_install_dir, patch_path.relative_to(module_dir))) + log.warning( + f"Failed to apply patch for module '{module_fullname}'. You will have to apply the patch manually" + ) + return False + + # Write the patched files to a temporary directory + log.debug("Writing patched files") + for file, new_content in new_files.items(): + fn = temp_module_dir / file + with open(fn, "w") as fh: + fh.writelines(new_content) + + # Create the new patch file + log.debug("Regenerating patch file") + ModulesDiffer.write_diff_file( + Path(temp_module_dir, patch_path.relative_to(module_dir)), + module, + repo_path, + module_install_dir, + temp_module_dir, + file_action="w", + for_git=False, + dsp_from_dir=module_relpath, + dsp_to_dir=module_relpath, + ) + + # Move the patched files to the install dir + log.debug("Overwriting installed files installed files with patched files") + shutil.rmtree(module_install_dir) + shutil.copytree(temp_module_dir, module_install_dir) + + # Add the patch file to the modules.json file + self.modules_json.add_patch_entry( + module, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=True + ) + + return True diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cc52ee896c..e6f4e03571 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -815,6 +815,29 @@ def get_modules_json(self): self.load() return copy.deepcopy(self.modules_json) + def get_component_version(self, component_type, component_name, repo_url, install_dir): + """ + Returns the version of a module or subworkflow + + Args: + component_name (str): Name of the module/subworkflow + repo_url (str): URL of the repository + install_dir (str): Name of the directory where modules/subworkflows are installed + + Returns: + (str): The git SHA of the module/subworkflow if it exists, None otherwise + """ + if self.modules_json is None: + self.load() + return ( + self.modules_json.get("repos", {}) + .get(repo_url, {}) + .get(component_type, {}) + .get(install_dir, {}) + .get(component_name, {}) + .get("git_sha", None) + ) + def get_module_version(self, module_name, repo_url, install_dir): """ Returns the version of a module diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 297e893401..1d1ef95379 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -1,25 +1,7 @@ -import logging -import os -import shutil -import tempfile -from pathlib import Path +from nf_core.components.update import ComponentUpdate -import questionary -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.components.components_utils import prompt_component_version_sha -from nf_core.utils import plural_es, plural_s, plural_y - -from .modules_differ import ModulesDiffer -from .modules_json import ModulesJson -from .modules_repo import ModulesRepo - -log = logging.getLogger(__name__) - - -class ModuleUpdate(ComponentCommand): +class ModuleUpdate(ComponentUpdate): def __init__( self, pipeline_dir, @@ -33,672 +15,16 @@ def __init__( branch=None, no_pull=False, ): - super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) - self.force = force - self.prompt = prompt - self.sha = sha - self.update_all = update_all - self.show_diff = show_diff - self.save_diff_fn = save_diff_fn - self.module = None - self.update_config = None - self.modules_json = ModulesJson(self.dir) - self.branch = branch - - def _parameter_checks(self): - """Checks the compatibilty of the supplied parameters. - - Raises: - UserWarning: if any checks fail. - """ - - if self.save_diff_fn and self.show_diff: - raise UserWarning("Either `--preview` or `--save_diff` can be specified, not both.") - - if self.update_all and self.module: - raise UserWarning("Either a module or the '--all' flag can be specified, not both.") - - if self.repo_type == "modules": - raise UserWarning("Modules in clones of nf-core/modules can not be updated.") - - if self.prompt and self.sha is not None: - raise UserWarning("Cannot use '--sha' and '--prompt' at the same time.") - - if not self.has_valid_directory(): - raise UserWarning("The command was not run in a valid pipeline directory.") - - def update(self, module=None): - """Updates a specified module or all modules modules in a pipeline. - - Args: - module (str): The name of the module to update. - - Returns: - bool: True if the update was successful, False otherwise. - """ - self.module = module - - tool_config = nf_core.utils.load_tools_config(self.dir) - self.update_config = tool_config.get("update", {}) - - self._parameter_checks() - - # Check modules directory structure - self.check_modules_structure() - - # Verify that 'modules.json' is consistent with the installed modules - self.modules_json.check_up_to_date() - - if not self.update_all and module is None: - choices = ["All modules", "Named module"] - self.update_all = ( - questionary.select( - "Update all modules or a single named module?", - choices=choices, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - == "All modules" - ) - - # Verify that the provided SHA exists in the repo - if self.sha is not None and not self.modules_repo.sha_exists_on_branch(self.sha): - log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") - return False - - # Get the list of modules to update, and their version information - modules_info = self.get_all_modules_info() if self.update_all else [self.get_single_module_info(module)] - - # Save the current state of the modules.json - old_modules_json = self.modules_json.get_modules_json() - - # Ask if we should show the diffs (unless a filename was already given on the command line) - if not self.save_diff_fn and self.show_diff is None: - diff_type = questionary.select( - "Do you want to view diffs of the proposed changes?", - choices=[ - {"name": "No previews, just update everything", "value": 0}, - {"name": "Preview diff in terminal, choose whether to update files", "value": 1}, - {"name": "Just write diffs to a patch file", "value": 2}, - ], - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - self.show_diff = diff_type == 1 - self.save_diff_fn = diff_type == 2 - - if self.save_diff_fn: # True or a string - self.setup_diff_file() - - # Loop through all modules to be updated - # and do the requested action on them - exit_value = True - all_patches_successful = True - for modules_repo, module, sha, patch_relpath in modules_info: - module_fullname = str(Path("modules", modules_repo.repo_path, module)) - # Are we updating the files in place or not? - dry_run = self.show_diff or self.save_diff_fn - - current_version = self.modules_json.get_module_version( - module, modules_repo.remote_url, modules_repo.repo_path - ) - - # Set the temporary installation folder - install_tmp_dir = Path(tempfile.mkdtemp()) - module_install_dir = install_tmp_dir / module - - # Compute the module directory - module_dir = os.path.join(self.dir, "modules", modules_repo.repo_path, module) - - if sha is not None: - version = sha - elif self.prompt: - version = prompt_component_version_sha( - module, "modules", modules_repo=modules_repo, installed_sha=current_version - ) - else: - version = modules_repo.get_latest_component_version(module, self.component_type) - - if current_version is not None and not self.force: - if current_version == version: - if self.sha or self.prompt: - log.info(f"'{module_fullname}' is already installed at {version}") - else: - log.info(f"'{module_fullname}' is already up to date") - continue - - # Download module files - if not self.install_component_files(module, version, modules_repo, install_tmp_dir): - exit_value = False - continue - - if patch_relpath is not None: - patch_successful = self.try_apply_patch( - module, modules_repo.repo_path, patch_relpath, module_dir, module_install_dir - ) - if patch_successful: - log.info(f"Module '{module_fullname}' patched successfully") - else: - log.warning(f"Failed to patch module '{module_fullname}'. Will proceed with unpatched files.") - all_patches_successful &= patch_successful - - if dry_run: - if patch_relpath is not None: - if patch_successful: - log.info("Current installation is compared against patched version in remote.") - else: - log.warning("Current installation is compared against unpatched version in remote.") - # Compute the diffs for the module - if self.save_diff_fn: - log.info(f"Writing diff file for module '{module_fullname}' to '{self.save_diff_fn}'") - ModulesDiffer.write_diff_file( - self.save_diff_fn, - module, - modules_repo.repo_path, - module_dir, - module_install_dir, - current_version, - version, - dsp_from_dir=module_dir, - dsp_to_dir=module_dir, - ) - - elif self.show_diff: - ModulesDiffer.print_diff( - module, - modules_repo.repo_path, - module_dir, - module_install_dir, - current_version, - version, - dsp_from_dir=module_dir, - dsp_to_dir=module_dir, - ) - - # Ask the user if they want to install the module - dry_run = not questionary.confirm( - f"Update module '{module}'?", default=False, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - - if not dry_run: - # Clear the module directory and move the installed files there - self.move_files_from_tmp_dir(module, install_tmp_dir, modules_repo.repo_path, version) - # Update modules.json with newly installed module - self.modules_json.update(modules_repo, module, version, self.component_type) - else: - # Don't save to a file, just iteratively update the variable - self.modules_json.update(modules_repo, module, version, self.component_type, write_file=False) - - if self.save_diff_fn: - # Write the modules.json diff to the file - ModulesDiffer.append_modules_json_diff( - self.save_diff_fn, - old_modules_json, - self.modules_json.get_modules_json(), - Path(self.dir, "modules.json"), - ) - if exit_value: - log.info( - f"[bold magenta italic] TIP! [/] If you are happy with the changes in '{self.save_diff_fn}', you " - "can apply them by running the command :point_right:" - f" [bold magenta italic]git apply {self.save_diff_fn} [/]" - ) - elif not all_patches_successful: - log.info(f"Updates complete. Please apply failed patch{plural_es(modules_info)} manually") - else: - log.info("Updates complete :sparkles:") - - return exit_value - - def get_single_module_info(self, module): - """Collects the module repository, version and sha for a module. - - Information about the module version in the '.nf-core.yml' overrides - the '--sha' option - - Args: - module_name (str): The name of the module to get info for. - - Returns: - (ModulesRepo, str, str): The modules repo containing the module, - the module name, and the module version. - - Raises: - LookupError: If the module is not found either in the pipeline or the modules repo. - UserWarning: If the '.nf-core.yml' entry is not valid. - """ - # Check if there are any modules installed from the repo - repo_url = self.modules_repo.remote_url - modules = self.modules_json.get_all_modules().get(repo_url) - choices = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] - if repo_url not in self.modules_json.get_all_modules(): - raise LookupError(f"No modules installed from '{repo_url}'") - - if module is None: - module = questionary.autocomplete( - "Tool name:", - choices=choices, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Get module installation directory - install_dir = [dir for dir, m in modules if module == m][0] - - # Check if module is installed before trying to update - if module not in choices: - raise LookupError(f"Module '{module}' is not installed in pipeline and could therefore not be updated") - - # Check that the supplied name is an available module - if module and module not in self.modules_repo.get_avail_components(self.component_type): - raise LookupError( - f"Module '{module}' not found in list of available modules." - f"Use the command 'nf-core modules list remote' to view available software" - ) - - sha = self.sha - if module in self.update_config.get(self.modules_repo.remote_url, {}).get(install_dir, {}): - # If the module to update is in .nf-core.yml config file - config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(module) - if config_entry is not None and config_entry is not True: - if config_entry is False: - raise UserWarning("Module's update entry in '.nf-core.yml' is set to False") - if not isinstance(config_entry, str): - raise UserWarning("Module's update entry in '.nf-core.yml' is of wrong type") - - sha = config_entry - if self.sha is not None: - log.warning( - f"Found entry in '.nf-core.yml' for module '{module}' " - "which will override version specified with '--sha'" - ) - else: - log.info(f"Found entry in '.nf-core.yml' for module '{module}'") - log.info(f"Updating module to ({sha})") - - # Check if the update branch is the same as the installation branch - current_branch = self.modules_json.get_component_branch( - self.component_type, module, self.modules_repo.remote_url, install_dir + super().__init__( + pipeline_dir, + "modules", + force, + prompt, + sha, + update_all, + show_diff, + save_diff_fn, + remote_url, + branch, + no_pull, ) - new_branch = self.modules_repo.branch - if current_branch != new_branch: - log.warning( - f"You are trying to update the '{Path(install_dir, module)}' module from " - f"the '{new_branch}' branch. This module was installed from the '{current_branch}'" - ) - switch = questionary.confirm(f"Do you want to update using the '{current_branch}' instead?").unsafe_ask() - if switch: - # Change the branch - self.modules_repo.setup_branch(current_branch) - - # If there is a patch file, get its filename - patch_fn = self.modules_json.get_patch_fn(module, self.modules_repo.remote_url, install_dir) - - return (self.modules_repo, module, sha, patch_fn) - - def get_all_modules_info(self, branch=None): - """Collects the module repository, version and sha for all modules. - - Information about the module version in the '.nf-core.yml' overrides the '--sha' option. - - Returns: - [(ModulesRepo, str, str)]: A list of tuples containing a ModulesRepo object, - the module name, and the module version. - """ - if branch is not None: - use_branch = questionary.confirm( - "'--branch' was specified. Should this branch be used to update all modules?", default=False - ) - if not use_branch: - branch = None - skipped_repos = [] - skipped_modules = [] - overridden_repos = [] - overridden_modules = [] - modules_info = {} - # Loop through all the modules in the pipeline - # and check if they have an entry in the '.nf-core.yml' file - for repo_name, modules in self.modules_json.get_all_modules().items(): - if repo_name not in self.update_config or self.update_config[repo_name] is True: - # There aren't restrictions for the repository in .nf-core.yml file - modules_info[repo_name] = {} - for module_dir, module in modules: - try: - modules_info[repo_name][module_dir].append( - ( - module, - self.sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ) - except KeyError: - modules_info[repo_name][module_dir] = [ - ( - module, - self.sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ] - elif isinstance(self.update_config[repo_name], dict): - # If it is a dict, then there are entries for individual modules or module directories - for module_dir in set([dir for dir, _ in modules]): - if isinstance(self.update_config[repo_name][module_dir], str): - # If a string is given it is the commit SHA to which we should update to - custom_sha = self.update_config[repo_name][module_dir] - modules_info[repo_name] = {} - for dir, module in modules: - if module_dir == dir: - try: - modules_info[repo_name][module_dir].append( - ( - module, - custom_sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ) - except KeyError: - modules_info[repo_name][module_dir] = [ - ( - module, - custom_sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ] - if self.sha is not None: - overridden_repos.append(repo_name) - elif self.update_config[repo_name][module_dir] is False: - for dir, module in modules: - if dir == module_dir: - skipped_modules.append(f"{module_dir}/{module}") - elif isinstance(self.update_config[repo_name][module_dir], dict): - # If it's a dict, there are entries for individual modules - dir_config = self.update_config[repo_name][module_dir] - modules_info[repo_name] = {} - for module_dir, module in modules: - if module not in dir_config or dir_config[module] is True: - try: - modules_info[repo_name][module_dir].append( - ( - module, - self.sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ) - except KeyError: - modules_info[repo_name][module_dir] = [ - ( - module, - self.sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ] - elif isinstance(dir_config[module], str): - # If a string is given it is the commit SHA to which we should update to - custom_sha = dir_config[module] - try: - modules_info[repo_name][module_dir].append( - ( - module, - custom_sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ) - except KeyError: - modules_info[repo_name][module_dir] = [ - ( - module, - custom_sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ] - if self.sha is not None: - overridden_modules.append(module) - elif dir_config[module] is False: - # Otherwise the entry must be 'False' and we should ignore the module - skipped_modules.append(f"{module_dir}/{module}") - else: - raise UserWarning( - f"Module '{module}' in '{module_dir}' has an invalid entry in '.nf-core.yml'" - ) - elif isinstance(self.update_config[repo_name], str): - # If a string is given it is the commit SHA to which we should update to - custom_sha = self.update_config[repo_name] - modules_info[repo_name] = {} - for module_dir, module in modules: - try: - modules_info[repo_name][module_dir].append( - ( - module, - custom_sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ) - except KeyError: - modules_info[repo_name][module_dir] = [ - ( - module, - custom_sha, - self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir - ), - ) - ] - if self.sha is not None: - overridden_repos.append(repo_name) - elif self.update_config[repo_name] is False: - skipped_repos.append(repo_name) - else: - raise UserWarning(f"Repo '{repo_name}' has an invalid entry in '.nf-core.yml'") - - if skipped_repos: - skipped_str = "', '".join(skipped_repos) - log.info(f"Skipping modules in repositor{plural_y(skipped_repos)}: '{skipped_str}'") - - if skipped_modules: - skipped_str = "', '".join(skipped_modules) - log.info(f"Skipping module{plural_s(skipped_modules)}: '{skipped_str}'") - - if overridden_repos: - overridden_str = "', '".join(overridden_repos) - log.info( - f"Overriding '--sha' flag for modules in repositor{plural_y(overridden_repos)} " - f"with '.nf-core.yml' entry: '{overridden_str}'" - ) - - if overridden_modules: - overridden_str = "', '".join(overridden_modules) - log.info( - f"Overriding '--sha' flag for module{plural_s(overridden_modules)} with " - f"'.nf-core.yml' entry: '{overridden_str}'" - ) - # Loop through modules_info and create on ModulesRepo object per remote and branch - repos_and_branches = {} - for repo_name, repo_content in modules_info.items(): - for module_dir, mods in repo_content.items(): - for mod, sha, mod_branch in mods: - if branch is not None: - mod_branch = branch - if (repo_name, mod_branch) not in repos_and_branches: - repos_and_branches[(repo_name, mod_branch)] = [] - repos_and_branches[(repo_name, mod_branch)].append((mod, sha)) - - # Create ModulesRepo objects - repo_objs_mods = [] - for (repo_url, branch), mods_shas in repos_and_branches.items(): - try: - modules_repo = ModulesRepo(remote_url=repo_url, branch=branch) - except LookupError as e: - log.warning(e) - log.info(f"Skipping modules in '{repo_url}'") - else: - repo_objs_mods.append((modules_repo, mods_shas)) - - # Flatten the list - modules_info = [(repo, mod, sha) for repo, mods_shas in repo_objs_mods for mod, sha in mods_shas] - - # Verify that that all modules and shas exist in their respective ModulesRepo, - # don't try to update those that don't - i = 0 - while i < len(modules_info): - repo, module, sha = modules_info[i] - if not repo.component_exists(module, self.component_type): - log.warning(f"Module '{module}' does not exist in '{repo.remote_url}'. Skipping...") - modules_info.pop(i) - elif sha is not None and not repo.sha_exists_on_branch(sha): - log.warning( - f"Git sha '{sha}' does not exists on the '{repo.branch}' of '{repo.remote_url}'. Skipping module '{module}'" - ) - modules_info.pop(i) - else: - i += 1 - - # Add patch filenames to the modules that have them - modules_info = [ - (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.remote_url, repo.repo_path)) - for repo, mod, sha in modules_info - ] - - return modules_info - - def setup_diff_file(self): - """Sets up the diff file. - - If the save diff option was chosen interactively, the user is asked to supply a name for the diff file. - - Then creates the file for saving the diff. - """ - if self.save_diff_fn is True: - # From questionary - no filename yet - self.save_diff_fn = questionary.path( - "Enter the filename: ", style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - - self.save_diff_fn = Path(self.save_diff_fn) - - # Check if filename already exists (questionary or cli) - while self.save_diff_fn.exists(): - if questionary.confirm(f"'{self.save_diff_fn}' exists. Remove file?").unsafe_ask(): - os.remove(self.save_diff_fn) - break - self.save_diff_fn = questionary.path( - "Enter a new filename: ", - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - self.save_diff_fn = Path(self.save_diff_fn) - - # This guarantees that the file exists after calling the function - self.save_diff_fn.touch() - - def move_files_from_tmp_dir(self, module, install_folder, repo_path, new_version): - """Move the files from the temporary to the installation directory. - - Args: - module (str): The module name. - install_folder [str]: The path to the temporary installation directory. - repo_path (str): The name of the directory where modules are installed - new_version (str): The version of the module that was installed. - """ - temp_module_dir = os.path.join(install_folder, module) - files = os.listdir(temp_module_dir) - pipeline_path = os.path.join(self.dir, "modules", repo_path, module) - - log.debug(f"Removing old version of module '{module}'") - self.clear_component_dir(module, pipeline_path) - - os.makedirs(pipeline_path) - for file in files: - path = os.path.join(temp_module_dir, file) - if os.path.exists(path): - shutil.move(path, os.path.join(pipeline_path, file)) - - log.info(f"Updating '{repo_path}/{module}'") - log.debug(f"Updating module '{module}' to {new_version} from {repo_path}") - - def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_install_dir): - """ - Try applying a patch file to the new module files - - - Args: - module (str): The name of the module - repo_path (str): The name of the repository where the module resides - patch_relpath (Path | str): The path to patch file in the pipeline - module_dir (Path | str): The module directory in the pipeline - module_install_dir (Path | str): The directory where the new module - file have been installed - - Returns: - (bool): Whether the patch application was successful - """ - module_fullname = str(Path(repo_path, module)) - log.info(f"Found patch for module '{module_fullname}'. Trying to apply it to new files") - - patch_path = Path(self.dir / patch_relpath) - module_relpath = Path("modules", repo_path, module) - - # Check that paths in patch file are updated - self.check_patch_paths(patch_path, module) - - # Copy the installed files to a new temporary directory to save them for later use - temp_dir = Path(tempfile.mkdtemp()) - temp_module_dir = temp_dir / module - shutil.copytree(module_install_dir, temp_module_dir) - - try: - new_files = ModulesDiffer.try_apply_patch(module, repo_path, patch_path, temp_module_dir) - except LookupError: - # Patch failed. Save the patch file by moving to the install dir - shutil.move(patch_path, Path(module_install_dir, patch_path.relative_to(module_dir))) - log.warning( - f"Failed to apply patch for module '{module_fullname}'. You will have to apply the patch manually" - ) - return False - - # Write the patched files to a temporary directory - log.debug("Writing patched files") - for file, new_content in new_files.items(): - fn = temp_module_dir / file - with open(fn, "w") as fh: - fh.writelines(new_content) - - # Create the new patch file - log.debug("Regenerating patch file") - ModulesDiffer.write_diff_file( - Path(temp_module_dir, patch_path.relative_to(module_dir)), - module, - repo_path, - module_install_dir, - temp_module_dir, - file_action="w", - for_git=False, - dsp_from_dir=module_relpath, - dsp_to_dir=module_relpath, - ) - - # Move the patched files to the install dir - log.debug("Overwriting installed files installed files with patched files") - shutil.rmtree(module_install_dir) - shutil.copytree(temp_module_dir, module_install_dir) - - # Add the patch file to the modules.json file - self.modules_json.add_patch_entry( - module, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=True - ) - - return True diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py new file mode 100644 index 0000000000..5272304c60 --- /dev/null +++ b/nf_core/subworkflows/update.py @@ -0,0 +1,30 @@ +from nf_core.components.update import ComponentUpdate + + +class SubworkflowUpdate(ComponentUpdate): + def __init__( + self, + pipeline_dir, + force=False, + prompt=False, + sha=None, + update_all=False, + show_diff=None, + save_diff_fn=None, + remote_url=None, + branch=None, + no_pull=False, + ): + super().__init__( + pipeline_dir, + "subworkflows", + force, + prompt, + sha, + update_all, + show_diff, + save_diff_fn, + remote_url, + branch, + no_pull, + ) From bde678c5adeef1d32553f667756992c61dbff8ea Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 10 Nov 2022 17:09:53 +0100 Subject: [PATCH 503/854] refactor components update --- nf_core/components/update.py | 404 ++++++++++++++++++----------------- 1 file changed, 212 insertions(+), 192 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 7ba6cff81e..a9db42e509 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -40,7 +40,7 @@ def __init__( self.update_all = update_all self.show_diff = show_diff self.save_diff_fn = save_diff_fn - self.module = None + self.component = None self.update_config = None self.modules_json = ModulesJson(self.dir) self.branch = branch @@ -55,11 +55,11 @@ def _parameter_checks(self): if self.save_diff_fn and self.show_diff: raise UserWarning("Either `--preview` or `--save_diff` can be specified, not both.") - if self.update_all and self.module: - raise UserWarning("Either a module or the '--all' flag can be specified, not both.") + if self.update_all and self.component: + raise UserWarning(f"Either a {self.component_type[:-1]} or the '--all' flag can be specified, not both.") if self.repo_type == "modules": - raise UserWarning("Modules in clones of nf-core/modules can not be updated.") + raise UserWarning(f"{self.component_type.title()} in clones of nf-core/modules can not be updated.") if self.prompt and self.sha is not None: raise UserWarning("Cannot use '--sha' and '--prompt' at the same time.") @@ -67,16 +67,19 @@ def _parameter_checks(self): if not self.has_valid_directory(): raise UserWarning("The command was not run in a valid pipeline directory.") - def update(self, module=None): - """Updates a specified module or all modules modules in a pipeline. + def update(self, component=None): + """Updates a specified module/subworkflow or all modules/subworkflows in a pipeline. + + If updating a subworkflow: updates all modules used in that subworkflow. + If updating a module: updates all subworkflows that use the module. Args: - module (str): The name of the module to update. + component (str): The name of the module/subworkflow to update. Returns: bool: True if the update was successful, False otherwise. """ - self.module = module + self.component = component tool_config = nf_core.utils.load_tools_config(self.dir) self.update_config = tool_config.get("update", {}) @@ -89,15 +92,15 @@ def update(self, module=None): # Verify that 'modules.json' is consistent with the installed modules self.modules_json.check_up_to_date() - if not self.update_all and module is None: - choices = ["All modules", "Named module"] + if not self.update_all and component is None: + choices = [f"All {self.component_type}", f"Named {self.component_type[:-1]}"] self.update_all = ( questionary.select( - "Update all modules or a single named module?", + f"Update all {self.component_type} or a single named {self.component_type[:-1]}?", choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - == "All modules" + == f"All {self.component_type}" ) # Verify that the provided SHA exists in the repo @@ -105,8 +108,10 @@ def update(self, module=None): log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'") return False - # Get the list of modules to update, and their version information - modules_info = self.get_all_modules_info() if self.update_all else [self.get_single_module_info(module)] + # Get the list of modules/subworkflows to update, and their version information + components_info = ( + self.get_all_components_info() if self.update_all else [self.get_single_component_info(component)] + ) # Save the current state of the modules.json old_modules_json = self.modules_json.get_modules_json() @@ -129,56 +134,58 @@ def update(self, module=None): if self.save_diff_fn: # True or a string self.setup_diff_file() - # Loop through all modules to be updated + # Loop through all components to be updated # and do the requested action on them exit_value = True all_patches_successful = True - for modules_repo, module, sha, patch_relpath in modules_info: - module_fullname = str(Path("modules", modules_repo.repo_path, module)) + for modules_repo, component, sha, patch_relpath in components_info: + component_fullname = str(Path(self.component_type, modules_repo.repo_path, component)) # Are we updating the files in place or not? dry_run = self.show_diff or self.save_diff_fn - current_version = self.modules_json.get_module_version( - module, modules_repo.remote_url, modules_repo.repo_path + current_version = self.modules_json.get_component_version( + self.component_type, component, modules_repo.remote_url, modules_repo.repo_path ) # Set the temporary installation folder install_tmp_dir = Path(tempfile.mkdtemp()) - module_install_dir = install_tmp_dir / module + component_install_dir = install_tmp_dir / component - # Compute the module directory - module_dir = os.path.join(self.dir, "modules", modules_repo.repo_path, module) + # Compute the component directory + component_dir = os.path.join(self.dir, self.component_type, modules_repo.repo_path, component) if sha is not None: version = sha elif self.prompt: version = prompt_component_version_sha( - module, "modules", modules_repo=modules_repo, installed_sha=current_version + component, self.component_type, modules_repo=modules_repo, installed_sha=current_version ) else: - version = modules_repo.get_latest_component_version(module, self.component_type) + version = modules_repo.get_latest_component_version(component, self.component_type) if current_version is not None and not self.force: if current_version == version: if self.sha or self.prompt: - log.info(f"'{module_fullname}' is already installed at {version}") + log.info(f"'{component_fullname}' is already installed at {version}") else: - log.info(f"'{module_fullname}' is already up to date") + log.info(f"'{component_fullname}' is already up to date") continue - # Download module files - if not self.install_component_files(module, version, modules_repo, install_tmp_dir): + # Download component files + if not self.install_component_files(component, version, modules_repo, install_tmp_dir): exit_value = False continue if patch_relpath is not None: patch_successful = self.try_apply_patch( - module, modules_repo.repo_path, patch_relpath, module_dir, module_install_dir + component, modules_repo.repo_path, patch_relpath, component_dir, component_install_dir ) if patch_successful: - log.info(f"Module '{module_fullname}' patched successfully") + log.info(f"{self.component_type[:-1].title()} '{component_fullname}' patched successfully") else: - log.warning(f"Failed to patch module '{module_fullname}'. Will proceed with unpatched files.") + log.warning( + f"Failed to patch {self.component_type[:-1]} '{component_fullname}'. Will proceed with unpatched files." + ) all_patches_successful &= patch_successful if dry_run: @@ -187,46 +194,50 @@ def update(self, module=None): log.info("Current installation is compared against patched version in remote.") else: log.warning("Current installation is compared against unpatched version in remote.") - # Compute the diffs for the module + # Compute the diffs for the component if self.save_diff_fn: - log.info(f"Writing diff file for module '{module_fullname}' to '{self.save_diff_fn}'") + log.info( + f"Writing diff file for {self.component_type[:-1]} '{component_fullname}' to '{self.save_diff_fn}'" + ) ModulesDiffer.write_diff_file( self.save_diff_fn, - module, + component, modules_repo.repo_path, - module_dir, - module_install_dir, + component_dir, + component_install_dir, current_version, version, - dsp_from_dir=module_dir, - dsp_to_dir=module_dir, + dsp_from_dir=component_dir, + dsp_to_dir=component_dir, ) elif self.show_diff: ModulesDiffer.print_diff( - module, - modules_repo.repo_path, - module_dir, - module_install_dir, + component, + components_repo.repo_path, + component_dir, + component_install_dir, current_version, version, - dsp_from_dir=module_dir, - dsp_to_dir=module_dir, + dsp_from_dir=component_dir, + dsp_to_dir=component_dir, ) - # Ask the user if they want to install the module + # Ask the user if they want to install the component dry_run = not questionary.confirm( - f"Update module '{module}'?", default=False, style=nf_core.utils.nfcore_question_style + f"Update {self.component_type[:-1]} '{component}'?", + default=False, + style=nf_core.utils.nfcore_question_style, ).unsafe_ask() if not dry_run: - # Clear the module directory and move the installed files there - self.move_files_from_tmp_dir(module, install_tmp_dir, modules_repo.repo_path, version) - # Update modules.json with newly installed module - self.modules_json.update(modules_repo, module, version, self.component_type) + # Clear the component directory and move the installed files there + self.move_files_from_tmp_dir(component, install_tmp_dir, modules_repo.repo_path, version) + # Update modules.json with newly installed component + self.modules_json.update(modules_repo, component, version, self.component_type) else: # Don't save to a file, just iteratively update the variable - self.modules_json.update(modules_repo, module, version, self.component_type, write_file=False) + self.modules_json.update(modules_repo, component, version, self.component_type, write_file=False) if self.save_diff_fn: # Write the modules.json diff to the file @@ -243,86 +254,92 @@ def update(self, module=None): f" [bold magenta italic]git apply {self.save_diff_fn} [/]" ) elif not all_patches_successful: - log.info(f"Updates complete. Please apply failed patch{plural_es(modules_info)} manually") + log.info(f"Updates complete. Please apply failed patch{plural_es(components_info)} manually") else: log.info("Updates complete :sparkles:") return exit_value - def get_single_module_info(self, module): - """Collects the module repository, version and sha for a module. + def get_single_component_info(self, component): + """Collects the modules repository, version and sha for a component. - Information about the module version in the '.nf-core.yml' overrides + Information about the component version in the '.nf-core.yml' overrides the '--sha' option Args: - module_name (str): The name of the module to get info for. + component (str): The name of the module/subworkflow to get info for. Returns: - (ModulesRepo, str, str): The modules repo containing the module, - the module name, and the module version. + (ModulesRepo, str, str): The modules repo containing the component, + the component name, and the component version. Raises: - LookupError: If the module is not found either in the pipeline or the modules repo. + LookupError: If the component is not found either in the pipeline or the modules repo. UserWarning: If the '.nf-core.yml' entry is not valid. """ - # Check if there are any modules installed from the repo + # Check if there are any modules/subworkflows installed from the repo repo_url = self.modules_repo.remote_url - modules = self.modules_json.get_all_modules().get(repo_url) - choices = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] - if repo_url not in self.modules_json.get_all_modules(): - raise LookupError(f"No modules installed from '{repo_url}'") - - if module is None: - module = questionary.autocomplete( - "Tool name:", + components = self.modules_json.get_all_components(self.component_type).get(repo_url) + choices = [component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components] + if repo_url not in self.modules_json.get_all_components(self.component_type): + raise LookupError(f"No {self.component_type} installed from '{repo_url}'") + + if component is None: + component = questionary.autocomplete( + f"{self.component_type[:-1].title()} name:", choices=choices, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - # Get module installation directory - install_dir = [dir for dir, m in modules if module == m][0] + # Get component installation directory + install_dir = [dir for dir, m in components if component == m][0] - # Check if module is installed before trying to update - if module not in choices: - raise LookupError(f"Module '{module}' is not installed in pipeline and could therefore not be updated") + # Check if component is installed before trying to update + if component not in choices: + raise LookupError( + f"{self.component_type[:-1].title()} '{component}' is not installed in pipeline and could therefore not be updated" + ) - # Check that the supplied name is an available module - if module and module not in self.modules_repo.get_avail_components(self.component_type): + # Check that the supplied name is an available module/subworkflow + if component and component not in self.modules_repo.get_avail_components(self.component_type): raise LookupError( - f"Module '{module}' not found in list of available modules." - f"Use the command 'nf-core modules list remote' to view available software" + f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." + f"Use the command 'nf-core {self.component_type} list remote' to view available software" ) sha = self.sha - if module in self.update_config.get(self.modules_repo.remote_url, {}).get(install_dir, {}): - # If the module to update is in .nf-core.yml config file - config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(module) + if component in self.update_config.get(self.modules_repo.remote_url, {}).get(install_dir, {}): + # If the component to update is in .nf-core.yml config file + config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(component) if config_entry is not None and config_entry is not True: if config_entry is False: - raise UserWarning("Module's update entry in '.nf-core.yml' is set to False") + raise UserWarning( + f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is set to False" + ) if not isinstance(config_entry, str): - raise UserWarning("Module's update entry in '.nf-core.yml' is of wrong type") + raise UserWarning( + f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is of wrong type" + ) sha = config_entry if self.sha is not None: log.warning( - f"Found entry in '.nf-core.yml' for module '{module}' " + f"Found entry in '.nf-core.yml' for {self.component_type[:-1]} '{component}' " "which will override version specified with '--sha'" ) else: - log.info(f"Found entry in '.nf-core.yml' for module '{module}'") - log.info(f"Updating module to ({sha})") + log.info(f"Found entry in '.nf-core.yml' for {self.component_type[:-1]} '{component}'") + log.info(f"Updating component to ({sha})") # Check if the update branch is the same as the installation branch current_branch = self.modules_json.get_component_branch( - self.component_type, module, self.modules_repo.remote_url, install_dir + self.component_type, component, self.modules_repo.remote_url, install_dir ) new_branch = self.modules_repo.branch if current_branch != new_branch: log.warning( - f"You are trying to update the '{Path(install_dir, module)}' module from " - f"the '{new_branch}' branch. This module was installed from the '{current_branch}'" + f"You are trying to update the '{Path(install_dir, component)}' {self.component_type[:-1]} from " + f"the '{new_branch}' branch. This {self.component_type[:-1]} was installed from the '{current_branch}'" ) switch = questionary.confirm(f"Do you want to update using the '{current_branch}' instead?").unsafe_ask() if switch: @@ -330,172 +347,173 @@ def get_single_module_info(self, module): self.modules_repo.setup_branch(current_branch) # If there is a patch file, get its filename - patch_fn = self.modules_json.get_patch_fn(module, self.modules_repo.remote_url, install_dir) + patch_fn = self.modules_json.get_patch_fn(component, self.modules_repo.remote_url, install_dir) - return (self.modules_repo, module, sha, patch_fn) + return (self.modules_repo, component, sha, patch_fn) - def get_all_modules_info(self, branch=None): - """Collects the module repository, version and sha for all modules. + def get_all_components_info(self, branch=None): + """Collects the modules repository, version and sha for all modules/subworkflows. - Information about the module version in the '.nf-core.yml' overrides the '--sha' option. + Information about the module/subworkflow version in the '.nf-core.yml' overrides the '--sha' option. Returns: [(ModulesRepo, str, str)]: A list of tuples containing a ModulesRepo object, - the module name, and the module version. + the component name, and the component version. """ if branch is not None: use_branch = questionary.confirm( - "'--branch' was specified. Should this branch be used to update all modules?", default=False + f"'--branch' was specified. Should this branch be used to update all {self.component_type}?", + default=False, ) if not use_branch: branch = None skipped_repos = [] - skipped_modules = [] + skipped_components = [] overridden_repos = [] - overridden_modules = [] - modules_info = {} - # Loop through all the modules in the pipeline + overridden_components = [] + components_info = {} + # Loop through all the modules/subworkflows in the pipeline # and check if they have an entry in the '.nf-core.yml' file - for repo_name, modules in self.modules_json.get_all_modules().items(): + for repo_name, components in self.modules_json.get_all_components(self.component_type).items(): if repo_name not in self.update_config or self.update_config[repo_name] is True: # There aren't restrictions for the repository in .nf-core.yml file - modules_info[repo_name] = {} - for module_dir, module in modules: + components_info[repo_name] = {} + for component_dir, component in components: try: - modules_info[repo_name][module_dir].append( + components_info[repo_name][component_dir].append( ( - module, + component, self.sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ) except KeyError: - modules_info[repo_name][module_dir] = [ + components_info[repo_name][component_dir] = [ ( - module, + component, self.sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ] elif isinstance(self.update_config[repo_name], dict): - # If it is a dict, then there are entries for individual modules or module directories - for module_dir in set([dir for dir, _ in modules]): - if isinstance(self.update_config[repo_name][module_dir], str): + # If it is a dict, then there are entries for individual components or component directories + for component_dir in set([dir for dir, _ in components]): + if isinstance(self.update_config[repo_name][component_dir], str): # If a string is given it is the commit SHA to which we should update to - custom_sha = self.update_config[repo_name][module_dir] - modules_info[repo_name] = {} - for dir, module in modules: - if module_dir == dir: + custom_sha = self.update_config[repo_name][component_dir] + components_info[repo_name] = {} + for dir, component in components: + if component_dir == dir: try: - modules_info[repo_name][module_dir].append( + components_info[repo_name][component_dir].append( ( - module, + component, custom_sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ) except KeyError: - modules_info[repo_name][module_dir] = [ + components_info[repo_name][component_dir] = [ ( - module, + component, custom_sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ] if self.sha is not None: overridden_repos.append(repo_name) - elif self.update_config[repo_name][module_dir] is False: - for dir, module in modules: - if dir == module_dir: - skipped_modules.append(f"{module_dir}/{module}") - elif isinstance(self.update_config[repo_name][module_dir], dict): - # If it's a dict, there are entries for individual modules - dir_config = self.update_config[repo_name][module_dir] - modules_info[repo_name] = {} - for module_dir, module in modules: - if module not in dir_config or dir_config[module] is True: + elif self.update_config[repo_name][component_dir] is False: + for dir, component in components: + if dir == component_dir: + skipped_components.append(f"{component_dir}/{components}") + elif isinstance(self.update_config[repo_name][component_dir], dict): + # If it's a dict, there are entries for individual components + dir_config = self.update_config[repo_name][component_dir] + components_info[repo_name] = {} + for component_dir, component in components: + if component not in dir_config or dir_config[component] is True: try: - modules_info[repo_name][module_dir].append( + components_info[repo_name][component_dir].append( ( - module, + component, self.sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ) except KeyError: - modules_info[repo_name][module_dir] = [ + components_info[repo_name][component_dir] = [ ( - module, + component, self.sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ] - elif isinstance(dir_config[module], str): + elif isinstance(dir_config[component], str): # If a string is given it is the commit SHA to which we should update to - custom_sha = dir_config[module] + custom_sha = dir_config[component] try: - modules_info[repo_name][module_dir].append( + components_info[repo_name][component_dir].append( ( - module, + component, custom_sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ) except KeyError: - modules_info[repo_name][module_dir] = [ + components_info[repo_name][component_dir] = [ ( - module, + component, custom_sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ] if self.sha is not None: - overridden_modules.append(module) - elif dir_config[module] is False: - # Otherwise the entry must be 'False' and we should ignore the module - skipped_modules.append(f"{module_dir}/{module}") + overridden_components.append(component) + elif dir_config[component] is False: + # Otherwise the entry must be 'False' and we should ignore the component + skipped_components.append(f"{component_dir}/{component}") else: raise UserWarning( - f"Module '{module}' in '{module_dir}' has an invalid entry in '.nf-core.yml'" + f"{self.component_type[:-1].title()} '{component}' in '{component_dir}' has an invalid entry in '.nf-core.yml'" ) elif isinstance(self.update_config[repo_name], str): # If a string is given it is the commit SHA to which we should update to custom_sha = self.update_config[repo_name] - modules_info[repo_name] = {} - for module_dir, module in modules: + components_info[repo_name] = {} + for component_dir, component in components: try: - modules_info[repo_name][module_dir].append( + components_info[repo_name][component_dir].append( ( - module, + component, custom_sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ) except KeyError: - modules_info[repo_name][module_dir] = [ + components_info[repo_name][component_dir] = [ ( - module, + component, custom_sha, self.modules_json.get_component_branch( - self.component_type, module, repo_name, module_dir + self.component_type, component, repo_name, component_dir ), ) ] @@ -508,73 +526,75 @@ def get_all_modules_info(self, branch=None): if skipped_repos: skipped_str = "', '".join(skipped_repos) - log.info(f"Skipping modules in repositor{plural_y(skipped_repos)}: '{skipped_str}'") + log.info(f"Skipping {self.component_type} in repositor{plural_y(skipped_repos)}: '{skipped_str}'") - if skipped_modules: - skipped_str = "', '".join(skipped_modules) - log.info(f"Skipping module{plural_s(skipped_modules)}: '{skipped_str}'") + if skipped_components: + skipped_str = "', '".join(skipped_components) + log.info(f"Skipping {self.component_type[:-1]}{plural_s(skipped_components)}: '{skipped_str}'") if overridden_repos: overridden_str = "', '".join(overridden_repos) log.info( - f"Overriding '--sha' flag for modules in repositor{plural_y(overridden_repos)} " + f"Overriding '--sha' flag for {self.component_type} in repositor{plural_y(overridden_repos)} " f"with '.nf-core.yml' entry: '{overridden_str}'" ) - if overridden_modules: - overridden_str = "', '".join(overridden_modules) + if overridden_components: + overridden_str = "', '".join(overridden_components) log.info( - f"Overriding '--sha' flag for module{plural_s(overridden_modules)} with " + f"Overriding '--sha' flag for {self.component_type[:-1]}{plural_s(overridden_components)} with " f"'.nf-core.yml' entry: '{overridden_str}'" ) - # Loop through modules_info and create on ModulesRepo object per remote and branch + # Loop through components_info and create on ModulesRepo object per remote and branch repos_and_branches = {} - for repo_name, repo_content in modules_info.items(): - for module_dir, mods in repo_content.items(): - for mod, sha, mod_branch in mods: + for repo_name, repo_content in components_info.items(): + for component_dir, comps in repo_content.items(): + for comp, sha, comp_branch in comps: if branch is not None: - mod_branch = branch - if (repo_name, mod_branch) not in repos_and_branches: - repos_and_branches[(repo_name, mod_branch)] = [] - repos_and_branches[(repo_name, mod_branch)].append((mod, sha)) + comp_branch = branch + if (repo_name, comp_branch) not in repos_and_branches: + repos_and_branches[(repo_name, comp_branch)] = [] + repos_and_branches[(repo_name, comp_branch)].append((comp, sha)) # Create ModulesRepo objects - repo_objs_mods = [] - for (repo_url, branch), mods_shas in repos_and_branches.items(): + repo_objs_comps = [] + for (repo_url, branch), comps_shas in repos_and_branches.items(): try: modules_repo = ModulesRepo(remote_url=repo_url, branch=branch) except LookupError as e: log.warning(e) - log.info(f"Skipping modules in '{repo_url}'") + log.info(f"Skipping {self.component_type} in '{repo_url}'") else: - repo_objs_mods.append((modules_repo, mods_shas)) + repo_objs_comps.append((modules_repo, comps_shas)) # Flatten the list - modules_info = [(repo, mod, sha) for repo, mods_shas in repo_objs_mods for mod, sha in mods_shas] + components_info = [(repo, comp, sha) for repo, comps_shas in repo_objs_comps for comp, sha in comps_shas] - # Verify that that all modules and shas exist in their respective ModulesRepo, + # Verify that that all components and shas exist in their respective ModulesRepo, # don't try to update those that don't i = 0 - while i < len(modules_info): - repo, module, sha = modules_info[i] - if not repo.component_exists(module, self.component_type): - log.warning(f"Module '{module}' does not exist in '{repo.remote_url}'. Skipping...") - modules_info.pop(i) + while i < len(components_info): + repo, component, sha = components_info[i] + if not repo.component_exists(component, self.component_type): + log.warning( + f"{self.component_type[:-1].title()} '{component}' does not exist in '{repo.remote_url}'. Skipping..." + ) + components_info.pop(i) elif sha is not None and not repo.sha_exists_on_branch(sha): log.warning( - f"Git sha '{sha}' does not exists on the '{repo.branch}' of '{repo.remote_url}'. Skipping module '{module}'" + f"Git sha '{sha}' does not exists on the '{repo.branch}' of '{repo.remote_url}'. Skipping {self.component_type[:-1]} '{component}'" ) - modules_info.pop(i) + components_info.pop(i) else: i += 1 - # Add patch filenames to the modules that have them - modules_info = [ - (repo, mod, sha, self.modules_json.get_patch_fn(mod, repo.remote_url, repo.repo_path)) - for repo, mod, sha in modules_info + # Add patch filenames to the components that have them + components_info = [ + (repo, comp, sha, self.modules_json.get_patch_fn(comp, repo.remote_url, repo.repo_path)) + for repo, comp, sha in components_info ] - return modules_info + return components_info def setup_diff_file(self): """Sets up the diff file. From c8f24948e9862bda64afde94753e2db4a60d5633 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 09:35:13 +0100 Subject: [PATCH 504/854] more refactoring of update command --- nf_core/components/update.py | 76 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index a9db42e509..17a583ac58 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -625,100 +625,100 @@ def setup_diff_file(self): # This guarantees that the file exists after calling the function self.save_diff_fn.touch() - def move_files_from_tmp_dir(self, module, install_folder, repo_path, new_version): + def move_files_from_tmp_dir(self, component, install_folder, repo_path, new_version): """Move the files from the temporary to the installation directory. Args: - module (str): The module name. + component (str): The module/subworkflow name. install_folder [str]: The path to the temporary installation directory. - repo_path (str): The name of the directory where modules are installed - new_version (str): The version of the module that was installed. + repo_path (str): The name of the directory where modules/subworkflows are installed + new_version (str): The version of the module/subworkflow that was installed. """ - temp_module_dir = os.path.join(install_folder, module) - files = os.listdir(temp_module_dir) - pipeline_path = os.path.join(self.dir, "modules", repo_path, module) + temp_component_dir = os.path.join(install_folder, component) + files = os.listdir(temp_component_dir) + pipeline_path = os.path.join(self.dir, self.component_type, repo_path, component) - log.debug(f"Removing old version of module '{module}'") - self.clear_component_dir(module, pipeline_path) + log.debug(f"Removing old version of {self.component_type[:-1]} '{component}'") + self.clear_component_dir(component, pipeline_path) os.makedirs(pipeline_path) for file in files: - path = os.path.join(temp_module_dir, file) + path = os.path.join(temp_component_dir, file) if os.path.exists(path): shutil.move(path, os.path.join(pipeline_path, file)) - log.info(f"Updating '{repo_path}/{module}'") - log.debug(f"Updating module '{module}' to {new_version} from {repo_path}") + log.info(f"Updating '{repo_path}/{component}'") + log.debug(f"Updating {self.component_type[:-1]} '{component}' to {new_version} from {repo_path}") - def try_apply_patch(self, module, repo_path, patch_relpath, module_dir, module_install_dir): + def try_apply_patch(self, component, repo_path, patch_relpath, component_dir, component_install_dir): """ - Try applying a patch file to the new module files + Try applying a patch file to the new module/subworkflow files Args: - module (str): The name of the module - repo_path (str): The name of the repository where the module resides + component (str): The name of the module/subworkflow + repo_path (str): The name of the repository where the module/subworkflow resides patch_relpath (Path | str): The path to patch file in the pipeline - module_dir (Path | str): The module directory in the pipeline - module_install_dir (Path | str): The directory where the new module - file have been installed + component_dir (Path | str): The module/subworkflow directory in the pipeline + component_install_dir (Path | str): The directory where the new component + file have been installed Returns: (bool): Whether the patch application was successful """ - module_fullname = str(Path(repo_path, module)) - log.info(f"Found patch for module '{module_fullname}'. Trying to apply it to new files") + component_fullname = str(Path(repo_path, component)) + log.info(f"Found patch for {self.component_type[:-1]} '{component_fullname}'. Trying to apply it to new files") patch_path = Path(self.dir / patch_relpath) - module_relpath = Path("modules", repo_path, module) + component_relpath = Path(self.component_type, repo_path, component) # Check that paths in patch file are updated - self.check_patch_paths(patch_path, module) + self.check_patch_paths(patch_path, component) # Copy the installed files to a new temporary directory to save them for later use temp_dir = Path(tempfile.mkdtemp()) - temp_module_dir = temp_dir / module - shutil.copytree(module_install_dir, temp_module_dir) + temp_component_dir = temp_dir / component + shutil.copytree(component_install_dir, temp_component_dir) try: - new_files = ModulesDiffer.try_apply_patch(module, repo_path, patch_path, temp_module_dir) + new_files = ModulesDiffer.try_apply_patch(component, repo_path, patch_path, temp_component_dir) except LookupError: # Patch failed. Save the patch file by moving to the install dir - shutil.move(patch_path, Path(module_install_dir, patch_path.relative_to(module_dir))) + shutil.move(patch_path, Path(component_install_dir, patch_path.relative_to(component_dir))) log.warning( - f"Failed to apply patch for module '{module_fullname}'. You will have to apply the patch manually" + f"Failed to apply patch for {self.component_type[:-1]} '{component_fullname}'. You will have to apply the patch manually" ) return False # Write the patched files to a temporary directory log.debug("Writing patched files") for file, new_content in new_files.items(): - fn = temp_module_dir / file + fn = temp_component_dir / file with open(fn, "w") as fh: fh.writelines(new_content) # Create the new patch file log.debug("Regenerating patch file") ModulesDiffer.write_diff_file( - Path(temp_module_dir, patch_path.relative_to(module_dir)), - module, + Path(temp_component_dir, patch_path.relative_to(component_dir)), + component, repo_path, - module_install_dir, - temp_module_dir, + component_install_dir, + temp_component_dir, file_action="w", for_git=False, - dsp_from_dir=module_relpath, - dsp_to_dir=module_relpath, + dsp_from_dir=component_relpath, + dsp_to_dir=component_relpath, ) # Move the patched files to the install dir log.debug("Overwriting installed files installed files with patched files") - shutil.rmtree(module_install_dir) - shutil.copytree(temp_module_dir, module_install_dir) + shutil.rmtree(component_install_dir) + shutil.copytree(temp_component_dir, component_install_dir) # Add the patch file to the modules.json file self.modules_json.add_patch_entry( - module, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=True + component, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=True ) return True From f29249cb4603426cf72a7e08151ab8ec0accbc0d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 09:47:43 +0100 Subject: [PATCH 505/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4dd46decb..a8e6733429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - `check_up_to_date()` function from `modules_json` also checks for subworkflows. - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) - Function create() from modules_json.py adds also subworkflows to modules.json file ([#2005](https://github.com/nf-core/tools/pull/2005)) +- Add `nf-core subworkflows update` command ([#2019](https://github.com/nf-core/tools/pull/2019)) ## [v2.6 - Tin Octopus](https://github.com/nf-core/tools/releases/tag/2.6) - [2022-10-04] From 3b15910939d2db680dc19993800c0bedfce5b8c7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 12:37:42 +0100 Subject: [PATCH 506/854] add tests for showing module diffs without updating --- tests/modules/update.py | 63 +++++++++++++++++++++++++++++++++++++++++ tests/test_modules.py | 2 ++ 2 files changed, 65 insertions(+) diff --git a/tests/modules/update.py b/tests/modules/update.py index 05f026fbbd..1aaf9f753a 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -2,13 +2,17 @@ import os import shutil import tempfile +from pathlib import Path +from unittest import mock +import questionary import yaml import nf_core.utils from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.modules.patch import ModulePatch from nf_core.modules.update import ModuleUpdate from ..utils import ( @@ -288,6 +292,65 @@ def test_update_different_branch_mix_modules_branch_test(self): assert modules_json.get_module_version("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA +# Mock questionary answer: do not update module, only show diffs +@mock.patch.object(questionary.Question, "unsafe_ask", return_value=False) +def test_update_only_show_differences(self, mock_prompt): + """Try updating all modules showing differences. + Don't update some of them. + Check that the sha in modules.json is not changed.""" + modules_json = ModulesJson(self.pipeline_dir) + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + + assert update_obj.update() is True + + mod_json = modules_json.get_modules_json() + # Loop through all modules and check that they are NOT updated (according to the modules.json file) + # Modules that can be updated but shouldn't are custom/dumpsoftwareversions and fastqc + # Module multiqc is already up to date so don't check + for mod in ["custom/dumpsoftwareversions", "fastqc"]: + correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + print(correct_git_sha, current_git_sha) + assert correct_git_sha != current_git_sha + + +# Mock questionary answer: do not update module, only show diffs +@mock.patch("questionary.confirm", side_effect=False) +def test_update_only_show_differences_when_patch(self): + """Try updating all modules showing differences when there's a patched module. + Don't update some of them. + Check that the sha in modules.json is not changed.""" + modules_json = ModulesJson(self.pipeline_dir) + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + + # Modify fastqc module, it will have a patch which will be applied during update + # We modify fastqc because it's one of the modules that can be updated and there's another one before it (custom/dumpsoftwareversions) + module_path = Path(self.pipeline_dir, "modules", "nf-core", "fastqc") + with open(module_path, "r") as fh: + lines = fh.readlines() + for line_index in range(len(lines)): + if lines[line_index] == " label 'process_medium'\n": + lines[line_index] = " label 'process_low'\n" + with open(module_path, "w") as fh: + fh.writelines(lines) + # Create a patch file + patch_obj = ModulePatch(self.pipeline_dir) + patch_obj.patch("fastqc") + patch_fn = "fastq.diff" + # Check that a patch file with the correct name has been created + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Update all modules + assert update_obj.update() is True + + mod_json = modules_json.get_modules_json() + # Loop through all modules and check that they are NOT updated (according to the modules.json file) + for mod in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: + correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + assert correct_git_sha != current_git_sha + + def cmp_module(dir1, dir2): """Compare two versions of the same module""" files = ["main.nf", "meta.yml"] diff --git a/tests/test_modules.py b/tests/test_modules.py index f14d256dc4..7d5288d781 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -200,6 +200,8 @@ def test_modulesrepo_class(self): test_update_different_branch_mix_modules_branch_test, test_update_different_branch_mixed_modules_main, test_update_different_branch_single_module, + test_update_only_show_differences, + test_update_only_show_differences_when_patch, test_update_with_config_dont_update, test_update_with_config_fix_all, test_update_with_config_fixed_version, From 963a0aacad2dd9d26dc1196b6a5803a9f48be25b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 12:53:34 +0100 Subject: [PATCH 507/854] fix patch --- tests/modules/update.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/modules/update.py b/tests/modules/update.py index 1aaf9f753a..a0ce20371c 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -310,13 +310,12 @@ def test_update_only_show_differences(self, mock_prompt): for mod in ["custom/dumpsoftwareversions", "fastqc"]: correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] - print(correct_git_sha, current_git_sha) assert correct_git_sha != current_git_sha # Mock questionary answer: do not update module, only show diffs -@mock.patch("questionary.confirm", side_effect=False) -def test_update_only_show_differences_when_patch(self): +@mock.patch.object(questionary.Question, "unsafe_ask", return_value=False) +def test_update_only_show_differences_when_patch(self, mock_prompt): """Try updating all modules showing differences when there's a patched module. Don't update some of them. Check that the sha in modules.json is not changed.""" @@ -326,28 +325,31 @@ def test_update_only_show_differences_when_patch(self): # Modify fastqc module, it will have a patch which will be applied during update # We modify fastqc because it's one of the modules that can be updated and there's another one before it (custom/dumpsoftwareversions) module_path = Path(self.pipeline_dir, "modules", "nf-core", "fastqc") - with open(module_path, "r") as fh: + main_path = Path(module_path, "main.nf") + with open(main_path, "r") as fh: lines = fh.readlines() for line_index in range(len(lines)): if lines[line_index] == " label 'process_medium'\n": lines[line_index] = " label 'process_low'\n" - with open(module_path, "w") as fh: + with open(main_path, "w") as fh: fh.writelines(lines) # Create a patch file patch_obj = ModulePatch(self.pipeline_dir) patch_obj.patch("fastqc") - patch_fn = "fastq.diff" # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "fastqc.diff"} # Update all modules assert update_obj.update() is True mod_json = modules_json.get_modules_json() # Loop through all modules and check that they are NOT updated (according to the modules.json file) - for mod in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: + # Modules that can be updated but shouldn't are custom/dumpsoftwareversions and fastqc + # Module multiqc is already up to date so don't check + for mod in ["custom/dumpsoftwareversions", "fastqc"]: correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + print(correct_git_sha, current_git_sha) assert correct_git_sha != current_git_sha From 3376ed22429bbc890df673de24dc2cda1fae28af Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 13:24:27 +0100 Subject: [PATCH 508/854] add subworkflows update command to main --- nf_core/__main__.py | 57 ++++++++++++++++++++++++++++++++ nf_core/subworkflows/__init__.py | 1 + 2 files changed, 58 insertions(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e7087b4931..377f006f2a 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1056,6 +1056,63 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin sys.exit(1) +# nf-core subworkflows update +@subworkflows.command() +@click.pass_context +@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help=r"Pipeline directory. [dim]\[default: current working directory][/]", +) +@click.option("-f", "--force", is_flag=True, default=False, help="Force update of subworkflow") +@click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the subworkflow") +@click.option("-s", "--sha", type=str, metavar="", help="Install subworkflow at commit SHA") +@click.option("-a", "--all", is_flag=True, default=False, help="Update all subworkflow installed in pipeline") +@click.option( + "-x/-y", + "--preview/--no-preview", + is_flag=True, + default=None, + help="Preview / no preview of changes before applying", +) +@click.option( + "-D", + "--save-diff", + type=str, + metavar="", + default=None, + help="Save diffs to a file instead of updating in place", +) +def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff): + """ + Update DSL2 subworkflow within a pipeline. + + Fetches and updates subworkflow files from a remote repo e.g. nf-core/modules. + """ + try: + subworkflow_install = nf_core.subworkflows.SubworkflowUpdate( + dir, + force, + prompt, + sha, + all, + preview, + save_diff, + ctx.obj["modules_repo_url"], + ctx.obj["modules_repo_branch"], + ctx.obj["modules_repo_no_pull"], + ) + exit_status = subworkflow_install.update(subworkflow) + if not exit_status and all: + sys.exit(1) + except (UserWarning, LookupError) as e: + log.error(e) + sys.exit(1) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index f10c850d8d..3a672c5a15 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -3,3 +3,4 @@ from .list import SubworkflowList from .subworkflows_test import SubworkflowsTest from .test_yml_builder import SubworkflowTestYmlBuilder +from .update import SubworkflowUpdate From bf1eb285ee70cd4c27382c488898a211ab3064ab Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 15:39:04 +0100 Subject: [PATCH 509/854] update modules and subworkflows recursively --- nf_core/__main__.py | 1 + nf_core/components/update.py | 63 +++++++++++++++++++++++++-- nf_core/modules/install.py | 4 +- nf_core/modules/modules_json.py | 77 ++++++++------------------------- nf_core/subworkflows/install.py | 11 +++-- 5 files changed, 88 insertions(+), 68 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 377f006f2a..1f6d266d19 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1109,6 +1109,7 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff): if not exit_status and all: sys.exit(1) except (UserWarning, LookupError) as e: + raise log.error(e) sys.exit(1) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 17a583ac58..05349231f8 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -214,7 +214,7 @@ def update(self, component=None): elif self.show_diff: ModulesDiffer.print_diff( component, - components_repo.repo_path, + modules_repo.repo_path, component_dir, component_install_dir, current_version, @@ -234,10 +234,14 @@ def update(self, component=None): # Clear the component directory and move the installed files there self.move_files_from_tmp_dir(component, install_tmp_dir, modules_repo.repo_path, version) # Update modules.json with newly installed component - self.modules_json.update(modules_repo, component, version, self.component_type) + self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type) + # Update linked components + self.update_linked_components(component, modules_repo.repo_path) else: # Don't save to a file, just iteratively update the variable - self.modules_json.update(modules_repo, component, version, self.component_type, write_file=False) + self.modules_json.update( + self.component_type, modules_repo, component, version, self.component_type, write_file=False + ) if self.save_diff_fn: # Write the modules.json diff to the file @@ -722,3 +726,56 @@ def try_apply_patch(self, component, repo_path, patch_relpath, component_dir, co ) return True + + def get_modules_subworkflows_to_update(self, component, install_dir): + """Get all modules and subworkflows linked to the updated component.""" + mods_json = self.modules_json.get_modules_json() + modules_to_update = [] + subworkflows_to_update = [] + installed_by = mods_json["repos"][self.modules_repo.remote_url][self.component_type][install_dir][component][ + "installed_by" + ] + + if self.component_type == "modules": + # All subworkflow names in the installed_by section of a module are subworkflows using this module + # We need to update them too + subworkflows_to_update = [subworkflow for subworkflow in installed_by if subworkflow != self.component_type] + elif self.component_type == "subworkflows": + for repo, repo_content in mods_json["repos"].items(): + for component_type, dir_content in repo_content.items(): + for dir, components in dir_content.items(): + for comp, comp_content in components.items(): + # If the updated subworkflow name appears in the installed_by section of the checked component + # The checked component is used by the updated subworkflow + # We need to update it too + if component in comp_content["installed_by"]: + if component_type == "modules": + modules_to_update.append(comp) + elif component_type == "subworkflows": + subworkflows_to_update.append(comp) + + return modules_to_update, subworkflows_to_update + + def update_linked_components(self, component, install_dir): + """ + Update modules and subworkflows linked to the component being updated. + """ + modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update(component, install_dir) + for s_update in subworkflows_to_update: + original_component_type = self._change_component_type("subworkflows") + self.update(s_update) + self._reset_component_type(original_component_type) + for m_update in modules_to_update: + original_component_type = self._change_component_type("modules") + self.update(m_update) + self._reset_component_type(original_component_type) + + def _change_component_type(self, new_component_type): + original_component_type = self.component_type + self.component_type = new_component_type + self.modules_json.pipeline_components = None + return original_component_type + + def _reset_component_type(self, original_component_type): + self.component_type = original_component_type + self.modules_json.pipeline_components = None diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 2a38f16c4c..346157d7a5 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -76,7 +76,7 @@ def install(self, module, silent=False): f"Module is already installed and force is not set.\nAdding the new installation source {self.installed_by} for module {module} to 'modules.json' without installing the module." ) modules_json.load() - modules_json.update(self.modules_repo, module, current_version, self.installed_by) + modules_json.update(self.component_type, self.modules_repo, module, current_version, self.installed_by) return False version = nf_core.components.components_install.get_version( @@ -110,5 +110,5 @@ def install(self, module, silent=False): # Update module.json with newly installed module modules_json.load() - modules_json.update(self.modules_repo, module, version, self.installed_by, install_track) + modules_json.update(self.component_type, self.modules_repo, module, version, self.installed_by, install_track) return True diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index e6f4e03571..7444e0e653 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -587,55 +587,15 @@ def load(self): except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") - def update(self, modules_repo, module_name, module_version, installed_by, installed_by_log=None, write_file=True): + def update(self, component_type, modules_repo, component_name, component_version, installed_by, installed_by_log=None, write_file=True): """ - Updates the 'module.json' file with new module info + Updates the 'module.json' file with new module/subworkflow info Args: - modules_repo (ModulesRepo): A ModulesRepo object configured for the new module - module_name (str): Name of new module - module_version (str): git SHA for the new module entry - installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json' - write_file (bool): whether to write the updated modules.json to a file. - """ - if installed_by_log is None: - installed_by_log = [] - - if self.modules_json is None: - self.load() - repo_name = modules_repo.repo_path - remote_url = modules_repo.remote_url - branch = modules_repo.branch - if remote_url not in self.modules_json["repos"]: - self.modules_json["repos"][remote_url] = {"modules": {repo_name: {}}} - repo_modules_entry = self.modules_json["repos"][remote_url]["modules"][repo_name] - if module_name not in repo_modules_entry: - repo_modules_entry[module_name] = {} - repo_modules_entry[module_name]["git_sha"] = module_version - repo_modules_entry[module_name]["branch"] = branch - try: - if installed_by not in repo_modules_entry[module_name]["installed_by"]: - repo_modules_entry[module_name]["installed_by"].append(installed_by) - except KeyError: - repo_modules_entry[module_name]["installed_by"] = [installed_by] - finally: - repo_modules_entry[module_name]["installed_by"].extend(installed_by_log) - - # Sort the 'modules.json' repo entries - self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - if write_file: - self.dump() - - def update_subworkflow( - self, modules_repo, subworkflow_name, subworkflow_version, installed_by, installed_by_log=None, write_file=True - ): - """ - Updates the 'module.json' file with new subworkflow info - - Args: - modules_repo (ModulesRepo): A ModulesRepo object configured for the new subworkflow - subworkflow_name (str): Name of new subworkflow - subworkflow_version (str): git SHA for the new subworkflow entry + component_type (str): modules or subworkflows + modules_repo (ModulesRepo): A ModulesRepo object configured for the new module/subworkflow + component_name (str): Name of new module/subworkflow + component_version (str): git SHA for the new module/subworkflow entry installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json' write_file (bool): whether to write the updated modules.json to a file. """ @@ -648,22 +608,21 @@ def update_subworkflow( remote_url = modules_repo.remote_url branch = modules_repo.branch if remote_url not in self.modules_json["repos"]: - self.modules_json["repos"][remote_url] = {"subworkflows": {repo_name: {}}} - if "subworkflows" not in self.modules_json["repos"][remote_url]: - # It's the first subworkflow installed in the pipeline! - self.modules_json["repos"][remote_url]["subworkflows"] = {repo_name: {}} - repo_subworkflows_entry = self.modules_json["repos"][remote_url]["subworkflows"][repo_name] - if subworkflow_name not in repo_subworkflows_entry: - repo_subworkflows_entry[subworkflow_name] = {} - repo_subworkflows_entry[subworkflow_name]["git_sha"] = subworkflow_version - repo_subworkflows_entry[subworkflow_name]["branch"] = branch + self.modules_json["repos"][remote_url] = {component_type: {repo_name: {}}} + if component_type not in self.modules_json["repos"][remote_url]: + self.modules_json["repos"][remote_url][component_type] = {repo_name: {}} + repo_component_entry = self.modules_json["repos"][remote_url][component_type][repo_name] + if component_name not in repo_component_entry: + repo_component_entry[component_name] = {} + repo_component_entry[component_name]["git_sha"] = component_version + repo_component_entry[component_name]["branch"] = branch try: - if installed_by not in repo_subworkflows_entry[subworkflow_name]["installed_by"]: - repo_subworkflows_entry[subworkflow_name]["installed_by"].append(installed_by) + if installed_by not in repo_component_entry[component_name]["installed_by"]: + repo_component_entry[component_name]["installed_by"].append(installed_by) except KeyError: - repo_subworkflows_entry[subworkflow_name]["installed_by"] = [installed_by] + repo_component_entry[component_name]["installed_by"] = [installed_by] finally: - repo_subworkflows_entry[subworkflow_name]["installed_by"].extend(installed_by_log) + repo_component_entry[component_name]["installed_by"].extend(installed_by_log) # Sort the 'modules.json' repo entries self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 2df0c48970..d16e37e6c1 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -82,7 +82,7 @@ def install(self, subworkflow, silent=False): f"Subworkflow is already installed and force is not set.\nAdding the new installation source {self.installed_by} for subworkflow {subworkflow} to 'modules.json' without installing the subworkflow." ) modules_json.load() - modules_json.update(self.modules_repo, subworkflow, current_version, self.installed_by) + modules_json.update(self.component_type, self.modules_repo, subworkflow, current_version, self.installed_by) return False version = nf_core.components.components_install.get_version( @@ -115,6 +115,12 @@ def install(self, subworkflow, silent=False): if not self.install_component_files(subworkflow, version, self.modules_repo, install_folder): return False + # Update module.json with newly installed subworkflow + modules_json.load() + modules_json.update( + self.component_type, self.modules_repo, subworkflow, version, self.installed_by, install_track + ) + # Install included modules and subworkflows self.install_included_components(subworkflow_dir) @@ -128,9 +134,6 @@ def install(self, subworkflow, silent=False): if os.path.isfile(subworkflow_config): log.info(f"Subworkflow config include statement: includeConfig '{subworkflow_config}'") - # Update module.json with newly installed subworkflow - modules_json.load() - modules_json.update_subworkflow(self.modules_repo, subworkflow, version, self.installed_by, install_track) return True def get_modules_subworkflows_to_install(self, subworkflow_dir): From 586883fbfb1ed924e8279dbff5cd2c174a6b9bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 11 Nov 2022 15:41:21 +0100 Subject: [PATCH 510/854] Update tests/modules/update.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- tests/modules/update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/modules/update.py b/tests/modules/update.py index a0ce20371c..b4b311308d 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -331,6 +331,7 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): for line_index in range(len(lines)): if lines[line_index] == " label 'process_medium'\n": lines[line_index] = " label 'process_low'\n" + break with open(main_path, "w") as fh: fh.writelines(lines) # Create a patch file From 6f335f3986c911d142e6cb1b088fcf2fc54f6287 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 16:17:09 +0100 Subject: [PATCH 511/854] add silent and updated arguments and add prompt --- nf_core/components/update.py | 37 ++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 05349231f8..3b56a2d865 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -67,7 +67,7 @@ def _parameter_checks(self): if not self.has_valid_directory(): raise UserWarning("The command was not run in a valid pipeline directory.") - def update(self, component=None): + def update(self, component=None, silent=False, updated=None): """Updates a specified module/subworkflow or all modules/subworkflows in a pipeline. If updating a subworkflow: updates all modules used in that subworkflow. @@ -80,6 +80,8 @@ def update(self, component=None): bool: True if the update was successful, False otherwise. """ self.component = component + if updated is None: + updated = [] tool_config = nf_core.utils.load_tools_config(self.dir) self.update_config = tool_config.get("update", {}) @@ -235,8 +237,22 @@ def update(self, component=None): self.move_files_from_tmp_dir(component, install_tmp_dir, modules_repo.repo_path, version) # Update modules.json with newly installed component self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type) - # Update linked components - self.update_linked_components(component, modules_repo.repo_path) + updated.append(component) + recursive_update = True + if not silent: + log.info( + f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be automatically updated." + "It is advised to keep all your modules and subworkflows up to date." + "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date." + ) + recursive_update = questionary.confirm( + "Would you like to continue updating all modules and subworkflows?", + default=True, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if recursive_update: + # Update linked components + self.update_linked_components(component, modules_repo.repo_path, updated) else: # Don't save to a file, just iteratively update the variable self.modules_json.update( @@ -257,9 +273,9 @@ def update(self, component=None): "can apply them by running the command :point_right:" f" [bold magenta italic]git apply {self.save_diff_fn} [/]" ) - elif not all_patches_successful: + elif not all_patches_successful and not silent: log.info(f"Updates complete. Please apply failed patch{plural_es(components_info)} manually") - else: + elif not silent: log.info("Updates complete :sparkles:") return exit_value @@ -756,18 +772,23 @@ def get_modules_subworkflows_to_update(self, component, install_dir): return modules_to_update, subworkflows_to_update - def update_linked_components(self, component, install_dir): + def update_linked_components(self, component, install_dir, updated=None): """ Update modules and subworkflows linked to the component being updated. """ modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update(component, install_dir) for s_update in subworkflows_to_update: + if s_update in updated: + continue original_component_type = self._change_component_type("subworkflows") - self.update(s_update) + self.update(s_update, silent=True, updated=updated) self._reset_component_type(original_component_type) + for m_update in modules_to_update: + if m_update in updated: + continue original_component_type = self._change_component_type("modules") - self.update(m_update) + self.update(m_update, silent=True, updated=updated) self._reset_component_type(original_component_type) def _change_component_type(self, new_component_type): From 03f5cdb2b8088ca343b64c5502c984a4e09ea68e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 16:59:39 +0100 Subject: [PATCH 512/854] add recursivity and warnings also for show diff and add diff to file --- nf_core/components/update.py | 72 +++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 3b56a2d865..10c3ff256d 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -67,7 +67,7 @@ def _parameter_checks(self): if not self.has_valid_directory(): raise UserWarning("The command was not run in a valid pipeline directory.") - def update(self, component=None, silent=False, updated=None): + def update(self, component=None, silent=False, updated=None, check_diff_exist=True): """Updates a specified module/subworkflow or all modules/subworkflows in a pipeline. If updating a subworkflow: updates all modules used in that subworkflow. @@ -134,7 +134,7 @@ def update(self, component=None, silent=False, updated=None): self.save_diff_fn = diff_type == 2 if self.save_diff_fn: # True or a string - self.setup_diff_file() + self.setup_diff_file(check_diff_exist) # Loop through all components to be updated # and do the requested action on them @@ -201,17 +201,41 @@ def update(self, component=None, silent=False, updated=None): log.info( f"Writing diff file for {self.component_type[:-1]} '{component_fullname}' to '{self.save_diff_fn}'" ) - ModulesDiffer.write_diff_file( - self.save_diff_fn, - component, - modules_repo.repo_path, - component_dir, - component_install_dir, - current_version, - version, - dsp_from_dir=component_dir, - dsp_to_dir=component_dir, - ) + try: + ModulesDiffer.write_diff_file( + self.save_diff_fn, + component, + modules_repo.repo_path, + component_dir, + component_install_dir, + current_version, + version, + dsp_from_dir=component_dir, + dsp_to_dir=component_dir, + ) + updated.append(component) + except UserWarning as e: + if str(e) != "Module is unchanged": + raise + else: + updated.append(component) + recursive_update = True + if not silent: + log.warning( + f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be added to the same diff file.\n" + "It is advised to keep all your modules and subworkflows up to date.\n" + "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" + ) + recursive_update = questionary.confirm( + "Would you like to continue adding all modules and subworkflows differences?", + default=True, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if recursive_update: + # Write all the differences of linked componenets to a diff file + self.update_linked_components( + component, modules_repo.repo_path, updated, check_diff_exist=False + ) elif self.show_diff: ModulesDiffer.print_diff( @@ -240,10 +264,10 @@ def update(self, component=None, silent=False, updated=None): updated.append(component) recursive_update = True if not silent: - log.info( - f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be automatically updated." - "It is advised to keep all your modules and subworkflows up to date." - "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date." + log.warning( + f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n" + "It is advised to keep all your modules and subworkflows up to date.\n" + "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" ) recursive_update = questionary.confirm( "Would you like to continue updating all modules and subworkflows?", @@ -267,7 +291,7 @@ def update(self, component=None, silent=False, updated=None): self.modules_json.get_modules_json(), Path(self.dir, "modules.json"), ) - if exit_value: + if exit_value and not silent: log.info( f"[bold magenta italic] TIP! [/] If you are happy with the changes in '{self.save_diff_fn}', you " "can apply them by running the command :point_right:" @@ -616,7 +640,7 @@ def get_all_components_info(self, branch=None): return components_info - def setup_diff_file(self): + def setup_diff_file(self, check_diff_exist=True): """Sets up the diff file. If the save diff option was chosen interactively, the user is asked to supply a name for the diff file. @@ -631,6 +655,10 @@ def setup_diff_file(self): self.save_diff_fn = Path(self.save_diff_fn) + if not check_diff_exist: + # This guarantees that the file exists after calling the function + self.save_diff_fn.touch() + return # Check if filename already exists (questionary or cli) while self.save_diff_fn.exists(): if questionary.confirm(f"'{self.save_diff_fn}' exists. Remove file?").unsafe_ask(): @@ -772,7 +800,7 @@ def get_modules_subworkflows_to_update(self, component, install_dir): return modules_to_update, subworkflows_to_update - def update_linked_components(self, component, install_dir, updated=None): + def update_linked_components(self, component, install_dir, updated=None, check_diff_exist=True): """ Update modules and subworkflows linked to the component being updated. """ @@ -781,14 +809,14 @@ def update_linked_components(self, component, install_dir, updated=None): if s_update in updated: continue original_component_type = self._change_component_type("subworkflows") - self.update(s_update, silent=True, updated=updated) + self.update(s_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) self._reset_component_type(original_component_type) for m_update in modules_to_update: if m_update in updated: continue original_component_type = self._change_component_type("modules") - self.update(m_update, silent=True, updated=updated) + self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) self._reset_component_type(original_component_type) def _change_component_type(self, new_component_type): From 841869097caa236d361dedc089705209095a510d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 17:05:48 +0100 Subject: [PATCH 513/854] handle update all --- nf_core/components/update.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 10c3ff256d..60de4a5aeb 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -808,23 +808,27 @@ def update_linked_components(self, component, install_dir, updated=None, check_d for s_update in subworkflows_to_update: if s_update in updated: continue - original_component_type = self._change_component_type("subworkflows") + original_component_type, original_update_all = self._change_component_type("subworkflows") self.update(s_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) - self._reset_component_type(original_component_type) + self._reset_component_type(original_component_type, original_update_all) for m_update in modules_to_update: if m_update in updated: continue - original_component_type = self._change_component_type("modules") + original_component_type, original_update_all = self._change_component_type("modules") self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) - self._reset_component_type(original_component_type) + self._reset_component_type(original_component_type, original_update_all) def _change_component_type(self, new_component_type): original_component_type = self.component_type self.component_type = new_component_type self.modules_json.pipeline_components = None - return original_component_type + # also reset update_all in case it's set + original_update_all = self.update_all + self.update_all = False + return original_component_type, original_update_all - def _reset_component_type(self, original_component_type): + def _reset_component_type(self, original_component_type, original_update_all): self.component_type = original_component_type self.modules_json.pipeline_components = None + self.update_all = original_update_all From 370c6cb212aa708e5ff74149790b70b8bdc87927 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 11 Nov 2022 17:11:13 +0100 Subject: [PATCH 514/854] don't prompt warning if update all --- nf_core/components/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 60de4a5aeb..a1fc4142f4 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -263,7 +263,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type) updated.append(component) recursive_update = True - if not silent: + if not silent and not self.update_all: log.warning( f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n" "It is advised to keep all your modules and subworkflows up to date.\n" From 9ce1c01f0304a79b7c96710b9d775c2e305999a1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 09:26:30 +0100 Subject: [PATCH 515/854] check that files are not updated --- tests/modules/update.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/modules/update.py b/tests/modules/update.py index a0ce20371c..5d4e219cbd 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -301,6 +301,10 @@ def test_update_only_show_differences(self, mock_prompt): modules_json = ModulesJson(self.pipeline_dir) update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + tmpdir = tempfile.mkdtemp() + shutil.rmtree(tmpdir) + shutil.copytree(Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME), tmpdir) + assert update_obj.update() is True mod_json = modules_json.get_modules_json() @@ -311,6 +315,7 @@ def test_update_only_show_differences(self, mock_prompt): correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] assert correct_git_sha != current_git_sha + assert cmp_module(Path(tmpdir, mod), Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, mod)) is True # Mock questionary answer: do not update module, only show diffs From 80fd5f6b72cdd49d5b3a8c0af522d83210d00854 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 09:30:07 +0100 Subject: [PATCH 516/854] run black --- nf_core/modules/modules_json.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 7444e0e653..ee7051a691 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -587,7 +587,16 @@ def load(self): except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") - def update(self, component_type, modules_repo, component_name, component_version, installed_by, installed_by_log=None, write_file=True): + def update( + self, + component_type, + modules_repo, + component_name, + component_version, + installed_by, + installed_by_log=None, + write_file=True, + ): """ Updates the 'module.json' file with new module/subworkflow info From c99f34f7fff74f8984807e439ea18a71447d7ce3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 10:30:47 +0100 Subject: [PATCH 517/854] take into account different repos --- nf_core/components/update.py | 19 +++++++++---------- tests/modules/modules_json.py | 4 ++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index a1fc4142f4..04eca86df0 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -233,9 +233,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr ).unsafe_ask() if recursive_update: # Write all the differences of linked componenets to a diff file - self.update_linked_components( - component, modules_repo.repo_path, updated, check_diff_exist=False - ) + self.update_linked_components(component, modules_repo, updated, check_diff_exist=False) elif self.show_diff: ModulesDiffer.print_diff( @@ -276,7 +274,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr ).unsafe_ask() if recursive_update: # Update linked components - self.update_linked_components(component, modules_repo.repo_path, updated) + self.update_linked_components(component, modules_repo, updated) else: # Don't save to a file, just iteratively update the variable self.modules_json.update( @@ -771,19 +769,20 @@ def try_apply_patch(self, component, repo_path, patch_relpath, component_dir, co return True - def get_modules_subworkflows_to_update(self, component, install_dir): + def get_modules_subworkflows_to_update(self, component, modules_repo): """Get all modules and subworkflows linked to the updated component.""" mods_json = self.modules_json.get_modules_json() modules_to_update = [] subworkflows_to_update = [] - installed_by = mods_json["repos"][self.modules_repo.remote_url][self.component_type][install_dir][component][ - "installed_by" - ] + installed_by = mods_json["repos"][modules_repo.remote_url][self.component_type][modules_repo.repo_path][ + component + ]["installed_by"] if self.component_type == "modules": # All subworkflow names in the installed_by section of a module are subworkflows using this module # We need to update them too subworkflows_to_update = [subworkflow for subworkflow in installed_by if subworkflow != self.component_type] + print(subworkflows_to_update) elif self.component_type == "subworkflows": for repo, repo_content in mods_json["repos"].items(): for component_type, dir_content in repo_content.items(): @@ -800,11 +799,11 @@ def get_modules_subworkflows_to_update(self, component, install_dir): return modules_to_update, subworkflows_to_update - def update_linked_components(self, component, install_dir, updated=None, check_diff_exist=True): + def update_linked_components(self, component, modules_repo, updated=None, check_diff_exist=True): """ Update modules and subworkflows linked to the component being updated. """ - modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update(component, install_dir) + modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update(component, modules_repo) for s_update in subworkflows_to_update: if s_update in updated: continue diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 20eee54e30..67ee44f973 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -32,7 +32,7 @@ def test_mod_json_update(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the modules.json file mod_repo_obj = ModulesRepo() - mod_json_obj.update(mod_repo_obj, "MODULE_NAME", "GIT_SHA", "modules", write_file=False) + mod_json_obj.update("modules", mod_repo_obj, "MODULE_NAME", "GIT_SHA", "modules", write_file=False) mod_json = mod_json_obj.get_modules_json() assert "MODULE_NAME" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"] assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"]["nf-core"]["MODULE_NAME"] @@ -155,7 +155,7 @@ def test_mod_json_up_to_date_reinstall_fails(self): mod_json_obj = ModulesJson(self.pipeline_dir) # Update the fastqc module entry to an invalid git_sha - mod_json_obj.update(ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", write_file=True) + mod_json_obj.update("modules", ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", write_file=True) # Remove the fastqc module fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") From 615b975afa179486c8d99a1ce17f3a7d85e5fc8f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 10:57:02 +0100 Subject: [PATCH 518/854] add argument to update recursively without prompts --- nf_core/__main__.py | 21 ++++++++++++++++++--- nf_core/components/update.py | 28 ++++++++++++++++++---------- nf_core/modules/modules_json.py | 2 ++ nf_core/modules/update.py | 2 ++ nf_core/subworkflows/update.py | 2 ++ tests/modules/patch.py | 4 ++-- tests/modules/update.py | 6 ++++-- 7 files changed, 48 insertions(+), 17 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 1f6d266d19..a42776d685 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -554,7 +554,14 @@ def install(ctx, tool, dir, prompt, force, sha): default=None, help="Save diffs to a file instead of updating in place", ) -def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff): +@click.option( + "-r", + "--recursive", + is_flat=True, + default=False, + help="Automatically update all linked modules and subworkflows without asking for confirmation", +) +def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, recursive): """ Update DSL2 modules within a pipeline. @@ -569,6 +576,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff): all, preview, save_diff, + recursive, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], @@ -1086,7 +1094,14 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin default=None, help="Save diffs to a file instead of updating in place", ) -def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff): +@click.option( + "-r", + "--recursive", + is_flat=True, + default=False, + help="Automatically update all linked modules and subworkflows without asking for confirmation", +) +def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, recursive): """ Update DSL2 subworkflow within a pipeline. @@ -1101,6 +1116,7 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff): all, preview, save_diff, + recursive, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], @@ -1109,7 +1125,6 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff): if not exit_status and all: sys.exit(1) except (UserWarning, LookupError) as e: - raise log.error(e) sys.exit(1) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 04eca86df0..e5c9d667cf 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -29,6 +29,7 @@ def __init__( update_all=False, show_diff=None, save_diff_fn=None, + recursive=False, remote_url=None, branch=None, no_pull=False, @@ -40,6 +41,7 @@ def __init__( self.update_all = update_all self.show_diff = show_diff self.save_diff_fn = save_diff_fn + self.recursive = recursive self.component = None self.update_config = None self.modules_json = ModulesJson(self.dir) @@ -226,11 +228,14 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr "It is advised to keep all your modules and subworkflows up to date.\n" "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" ) - recursive_update = questionary.confirm( - "Would you like to continue adding all modules and subworkflows differences?", - default=True, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() + if self.recursive: + recursive_update = True + else: + recursive_update = questionary.confirm( + "Would you like to continue adding all modules and subworkflows differences?", + default=True, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() if recursive_update: # Write all the differences of linked componenets to a diff file self.update_linked_components(component, modules_repo, updated, check_diff_exist=False) @@ -267,11 +272,14 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr "It is advised to keep all your modules and subworkflows up to date.\n" "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" ) - recursive_update = questionary.confirm( - "Would you like to continue updating all modules and subworkflows?", - default=True, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() + if self.recursive: + recursive_update = True + else: + recursive_update = questionary.confirm( + "Would you like to continue updating all modules and subworkflows?", + default=True, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() if recursive_update: # Update linked components self.update_linked_components(component, modules_repo, updated) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index ee7051a691..f636ac562d 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -459,6 +459,8 @@ def has_git_url_and_modules(self): """ for repo_url, repo_entry in self.modules_json.get("repos", {}).items(): if "modules" not in repo_entry: + if "subworkflows" in repo_entry: + continue log.warning(f"modules.json entry {repo_entry} does not have a modules entry") return False elif ( diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 1d1ef95379..350401c43e 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -11,6 +11,7 @@ def __init__( update_all=False, show_diff=None, save_diff_fn=None, + recursive=False, remote_url=None, branch=None, no_pull=False, @@ -24,6 +25,7 @@ def __init__( update_all, show_diff, save_diff_fn, + recursive, remote_url, branch, no_pull, diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py index 5272304c60..200ade9c50 100644 --- a/nf_core/subworkflows/update.py +++ b/nf_core/subworkflows/update.py @@ -11,6 +11,7 @@ def __init__( update_all=False, show_diff=None, save_diff_fn=None, + recursive=False, remote_url=None, branch=None, no_pull=False, @@ -24,6 +25,7 @@ def __init__( update_all, show_diff, save_diff_fn, + recursive, remote_url, branch, no_pull, diff --git a/tests/modules/patch.py b/tests/modules/patch.py index f1597cd776..100afe8731 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -235,7 +235,7 @@ def test_create_patch_update_success(self): # Update the module update_obj = nf_core.modules.ModuleUpdate( - self.pipeline_dir, sha=SUCCEED_SHA, show_diff=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH + self.pipeline_dir, sha=SUCCEED_SHA, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=PATCH_BRANCH ) assert update_obj.update(BISMARK_ALIGN) @@ -294,7 +294,7 @@ def test_create_patch_update_fail(self): patch_contents = fh.read() update_obj = nf_core.modules.ModuleUpdate( - self.pipeline_dir, sha=FAIL_SHA, show_diff=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH + self.pipeline_dir, sha=FAIL_SHA, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=PATCH_BRANCH ) update_obj.update(BISMARK_ALIGN) diff --git a/tests/modules/update.py b/tests/modules/update.py index 05f026fbbd..60f869ae4c 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -41,7 +41,9 @@ def test_install_and_update(self): def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" assert self.mods_install_old.install("trimgalore") - update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH) + update_obj = ModuleUpdate( + self.pipeline_dir, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + ) # Copy the module files and check that they are affected by the update tmpdir = tempfile.mkdtemp() @@ -225,7 +227,7 @@ def test_update_different_branch_single_module(self): assert install_obj.install("fastp") update_obj = ModuleUpdate( - self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False + self.pipeline_dir, recursive=True, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False ) update_obj.update("fastp") From 544a4ff7b821f2d53ad4c074c3760390b07ac34c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 10:58:32 +0100 Subject: [PATCH 519/854] remove print --- nf_core/components/update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index e5c9d667cf..582484589d 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -790,7 +790,6 @@ def get_modules_subworkflows_to_update(self, component, modules_repo): # All subworkflow names in the installed_by section of a module are subworkflows using this module # We need to update them too subworkflows_to_update = [subworkflow for subworkflow in installed_by if subworkflow != self.component_type] - print(subworkflows_to_update) elif self.component_type == "subworkflows": for repo, repo_content in mods_json["repos"].items(): for component_type, dir_content in repo_content.items(): From 5874a0b65f1ed704f3506ebf18b07b3bba399a5d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 11:11:21 +0100 Subject: [PATCH 520/854] merge from dev --- nf_core/components/update.py | 11 +- tests/subworkflows/update.py | 296 +++++++++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 tests/subworkflows/update.py diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 582484589d..4648a31875 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -182,7 +182,12 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr if patch_relpath is not None: patch_successful = self.try_apply_patch( - component, modules_repo.repo_path, patch_relpath, component_dir, component_install_dir + component, + modules_repo.repo_path, + patch_relpath, + component_dir, + component_install_dir, + write_file=False, ) if patch_successful: log.info(f"{self.component_type[:-1].title()} '{component_fullname}' patched successfully") @@ -704,7 +709,7 @@ def move_files_from_tmp_dir(self, component, install_folder, repo_path, new_vers log.info(f"Updating '{repo_path}/{component}'") log.debug(f"Updating {self.component_type[:-1]} '{component}' to {new_version} from {repo_path}") - def try_apply_patch(self, component, repo_path, patch_relpath, component_dir, component_install_dir): + def try_apply_patch(self, component, repo_path, patch_relpath, component_dir, component_install_dir, write_file=True): """ Try applying a patch file to the new module/subworkflow files @@ -772,7 +777,7 @@ def try_apply_patch(self, component, repo_path, patch_relpath, component_dir, co # Add the patch file to the modules.json file self.modules_json.add_patch_entry( - component, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=True + component, self.modules_repo.remote_url, repo_path, patch_relpath, write_file=write_file ) return True diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py new file mode 100644 index 0000000000..60f869ae4c --- /dev/null +++ b/tests/subworkflows/update.py @@ -0,0 +1,296 @@ +import filecmp +import os +import shutil +import tempfile + +import yaml + +import nf_core.utils +from nf_core.modules.install import ModuleInstall +from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.modules.update import ModuleUpdate + +from ..utils import ( + GITLAB_BRANCH_TEST_BRANCH, + GITLAB_BRANCH_TEST_NEW_SHA, + GITLAB_BRANCH_TEST_OLD_SHA, + GITLAB_DEFAULT_BRANCH, + GITLAB_REPO, + GITLAB_URL, + OLD_TRIMGALORE_BRANCH, + OLD_TRIMGALORE_SHA, +) + + +def test_install_and_update(self): + """Installs a module in the pipeline and updates it (no change)""" + self.mods_install.install("trimgalore") + update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) + + # Copy the module files and check that they are unaffected by the update + tmpdir = tempfile.mkdtemp() + trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") + shutil.copytree(trimgalore_path, trimgalore_tmpdir) + + assert update_obj.update("trimgalore") is True + assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + + +def test_install_at_hash_and_update(self): + """Installs an old version of a module in the pipeline and updates it""" + assert self.mods_install_old.install("trimgalore") + update_obj = ModuleUpdate( + self.pipeline_dir, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + ) + + # Copy the module files and check that they are affected by the update + tmpdir = tempfile.mkdtemp() + trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") + shutil.copytree(trimgalore_path, trimgalore_tmpdir) + + assert update_obj.update("trimgalore") is True + assert cmp_module(trimgalore_tmpdir, trimgalore_path) is False + + # Check that the modules.json is correctly updated + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + # Get the up-to-date git_sha for the module from the ModulesRepo object + correct_git_sha = update_obj.modules_repo.get_latest_component_version("trimgalore", "modules") + current_git_sha = mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] + assert correct_git_sha == current_git_sha + + +def test_install_at_hash_and_update_and_save_diff_to_file(self): + """Installs an old version of a module in the pipeline and updates it""" + self.mods_install_old.install("trimgalore") + patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") + update_obj = ModuleUpdate( + self.pipeline_dir, + save_diff_fn=patch_path, + sha=OLD_TRIMGALORE_SHA, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + ) + + # Copy the module files and check that they are affected by the update + tmpdir = tempfile.mkdtemp() + trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") + trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") + shutil.copytree(trimgalore_path, trimgalore_tmpdir) + + assert update_obj.update("trimgalore") is True + assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + + # TODO: Apply the patch to the module + + +def test_update_all(self): + """Updates all modules present in the pipeline""" + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + # Get the current modules.json + assert update_obj.update() is True + + # We must reload the modules.json to get the updated version + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json = mod_json_obj.get_modules_json() + # Loop through all modules and check that they are updated (according to the modules.json file) + for mod in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: + correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + assert correct_git_sha == current_git_sha + + +def test_update_with_config_fixed_version(self): + """Try updating when there are entries in the .nf-core.yml""" + # Install trimgalore at the latest version + assert self.mods_install_trimgalore.install("trimgalore") + + # Fix the trimgalore version in the .nf-core.yml to an old version + update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": OLD_TRIMGALORE_SHA}}} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + ) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] + assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + + +def test_update_with_config_dont_update(self): + """Try updating when module is to be ignored""" + # Install an old version of trimgalore + self.mods_install_old.install("trimgalore") + + # Set the trimgalore field to no update in the .nf-core.yml + update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": False}}} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate( + self.pipeline_dir, + update_all=True, + show_diff=False, + sha=OLD_TRIMGALORE_SHA, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + ) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] + assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + + +def test_update_with_config_fix_all(self): + """Fix the version of all nf-core modules""" + self.mods_install_trimgalore.install("trimgalore") + + # Fix the version of all nf-core modules in the .nf-core.yml to an old version + update_config = {GITLAB_URL: OLD_TRIMGALORE_SHA} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + ) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] + assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + + +def test_update_with_config_no_updates(self): + """Don't update any nf-core modules""" + self.mods_install_old.install("trimgalore") + old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + + # Fix the version of all nf-core modules in the .nf-core.yml to an old version + update_config = {GITLAB_URL: False} + tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) + tools_config["update"] = update_config + with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + yaml.dump(tools_config, f) + + # Update all modules in the pipeline + update_obj = ModuleUpdate( + self.pipeline_dir, + update_all=True, + show_diff=False, + sha=OLD_TRIMGALORE_SHA, + remote_url=GITLAB_URL, + branch=OLD_TRIMGALORE_BRANCH, + ) + assert update_obj.update() is True + + # Check that the git sha for trimgalore is correctly downgraded and none of the modules has changed + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + for module in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]: + assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module] + assert ( + mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module]["git_sha"] + == old_mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module]["git_sha"] + ) + + +def test_update_different_branch_single_module(self): + """Try updating a module in a specific branch""" + install_obj = ModuleInstall( + self.pipeline_dir, + prompt=False, + force=False, + remote_url=GITLAB_URL, + branch=GITLAB_BRANCH_TEST_BRANCH, + sha=GITLAB_BRANCH_TEST_OLD_SHA, + ) + assert install_obj.install("fastp") + + update_obj = ModuleUpdate( + self.pipeline_dir, recursive=True, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False + ) + update_obj.update("fastp") + + # Verify that the branch entry was updated correctly + modules_json = ModulesJson(self.pipeline_dir) + assert ( + modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) + == GITLAB_BRANCH_TEST_BRANCH + ) + assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + + +def test_update_different_branch_mixed_modules_main(self): + """Try updating all modules where MultiQC is installed from main branch""" + # Install fastp + assert self.mods_install_gitlab_old.install("fastp") + + # Install MultiQC from gitlab default branch + assert self.mods_install_gitlab.install("multiqc") + + # Try updating + update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) + assert update_obj.update() is True + + modules_json = ModulesJson(self.pipeline_dir) + # Verify that the branch entry was updated correctly + assert ( + modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) + == GITLAB_BRANCH_TEST_BRANCH + ) + assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + # MultiQC is present in both branches but should've been updated using the 'main' branch + assert ( + modules_json.get_component_branch(self.component_type, "multiqc", GITLAB_URL, GITLAB_REPO) + == GITLAB_DEFAULT_BRANCH + ) + + +def test_update_different_branch_mix_modules_branch_test(self): + """Try updating all modules where MultiQC is installed from branch-test branch""" + # Install multiqc from the branch-test branch + assert self.mods_install_gitlab_old.install( + "multiqc" + ) # Force as the same module is installed from github nf-core modules repo + modules_json = ModulesJson(self.pipeline_dir) + update_obj = ModuleUpdate( + self.pipeline_dir, + update_all=True, + show_diff=False, + remote_url=GITLAB_URL, + branch=GITLAB_BRANCH_TEST_BRANCH, + sha=GITLAB_BRANCH_TEST_NEW_SHA, + ) + assert update_obj.update() + + assert ( + modules_json.get_component_branch(self.component_type, "multiqc", GITLAB_URL, GITLAB_REPO) + == GITLAB_BRANCH_TEST_BRANCH + ) + assert modules_json.get_module_version("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA + + +def cmp_module(dir1, dir2): + """Compare two versions of the same module""" + files = ["main.nf", "meta.yml"] + return all(filecmp.cmp(os.path.join(dir1, f), os.path.join(dir2, f), shallow=False) for f in files) From 9c95c5c33ad8f849cc171d6cd3ee7f7607964e32 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 14 Nov 2022 12:37:04 +0100 Subject: [PATCH 521/854] make running prettier return better error messages --- nf_core/lint_utils.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 7f904b756c..31519f9dc6 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -76,12 +76,19 @@ def run_prettier_on_file(file): def _run_prettier_on_file(file): """Run natively installed Prettier on a file.""" - subprocess.run( - ["prettier", "--write", file], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) + try: + subprocess.run( + ["prettier", "--write", file], + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as e: + if ": SyntaxError: " in e.stderr.decode(): + raise ValueError(f"Can't format {file} because it has a synthax error.\n{e.stderr.decode()}") from e + raise ValueError( + "There was an error running the prettier pre-commit hook.\n" + f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" + ) from e def _run_pre_commit_prettier_on_file(file): @@ -95,10 +102,16 @@ def _run_pre_commit_prettier_on_file(file): """ nf_core_pre_commit_config = Path(nf_core.__file__).parent.parent / ".pre-commit-config.yaml" - - subprocess.run( - ["pre-commit", "run", f"--config {nf_core_pre_commit_config}", "prettier", f"--files {file}"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) + try: + subprocess.run( + ["pre-commit", "run", "--config", nf_core_pre_commit_config, "prettier", "--files", file], + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError as e: + if ": SyntaxError: " in e.stdout.decode(): + raise ValueError(f"Can't format {file} because it has a synthax error.\n{e.stdout.decode()}") from e + raise ValueError( + "There was an error running the prettier pre-commit hook.\n" + f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" + ) from e From 22819d9340f998546da38ed2349d5ff77fc4af89 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 14 Nov 2022 12:37:26 +0100 Subject: [PATCH 522/854] add tests for prettier invocations --- tests/test_lint_utils.py | 100 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 tests/test_lint_utils.py diff --git a/tests/test_lint_utils.py b/tests/test_lint_utils.py new file mode 100644 index 0000000000..1d864d77df --- /dev/null +++ b/tests/test_lint_utils.py @@ -0,0 +1,100 @@ +import shutil +from pathlib import Path +from unittest import mock + +import git +import pytest + +import nf_core.lint_utils + +JSON_WITH_SYNTAX_ERROR = "{'a':1, 1}" +JSON_MALFORMED = "{'a':1}" +JSON_FORMATTED = '{ "a": 1 }\n' + +WHICH_PRE_COMMIT = shutil.which("pre-commit") + + +@pytest.fixture() +def temp_git_repo(tmp_path_factory): + tmp_git_dir = tmp_path_factory.mktemp("tmp_git_dir") + repo = git.Repo.init(tmp_git_dir) + return tmp_git_dir, repo + + +@pytest.fixture(name="formatted_json") +def git_dir_with_json(temp_git_repo): + tmp_git_dir, repo = temp_git_repo + file = tmp_git_dir / "formatted.json" + with open(file, "w", encoding="utf-8") as f: + f.write(JSON_FORMATTED) + repo.git.add(file) + return file + + +@pytest.fixture(name="malformed_json") +def git_dir_with_json_malformed(temp_git_repo): + tmp_git_dir, repo = temp_git_repo + file = tmp_git_dir / "malformed.json" + with open(file, "w", encoding="utf-8") as f: + f.write(JSON_MALFORMED) + repo.git.add(file) + return file + + +@pytest.fixture(name="synthax_error_json") +def git_dir_with_json_syntax_error(temp_git_repo): + tmp_git_dir, repo = temp_git_repo + file = tmp_git_dir / "synthax-error.json" + with open(file, "w", encoding="utf-8") as f: + f.write(JSON_WITH_SYNTAX_ERROR) + repo.git.add(file) + return file + + +@pytest.fixture +def ensure_prettier_is_not_found(monkeypatch): + def dont_find_prettier(x): + if x == "pre-commit": + which_x = WHICH_PRE_COMMIT + elif x == "prettier": + which_x = None + else: + raise ValueError(f"This mock is only inteded to hide prettier form tests. {x}") + return which_x + + monkeypatch.setattr("nf_core.lint_utils.shutil.which", dont_find_prettier) + + +@pytest.mark.skipif(shutil.which("prettier") is None, reason="Can't test prettier if it is not available.") +def test_run_prettier_on_formatted_file(formatted_json): + nf_core.lint_utils.run_prettier_on_file(formatted_json) + assert formatted_json.read_text() == JSON_FORMATTED + + +def test_run_pre_commit_prettier_on_formatted_file(formatted_json, ensure_prettier_is_not_found): + nf_core.lint_utils.run_prettier_on_file(formatted_json) + assert formatted_json.read_text() == JSON_FORMATTED + + +@pytest.mark.skipif(shutil.which("prettier") is None, reason="Can't test prettier if it is not available.") +def test_run_prettier_on_malformed_file(malformed_json): + nf_core.lint_utils.run_prettier_on_file(malformed_json) + assert malformed_json.read_text() == JSON_FORMATTED + + +def test_run_prettier_pre_commit_on_malformed_file(malformed_json, ensure_prettier_is_not_found): + nf_core.lint_utils.run_prettier_on_file(malformed_json) + assert malformed_json.read_text() == JSON_FORMATTED + + +@pytest.mark.skipif(shutil.which("prettier") is None, reason="Can't test prettier if it is not available.") +def test_run_prettier_on_synthax_error_file(synthax_error_json): + with pytest.raises(ValueError) as exc_info: + nf_core.lint_utils.run_prettier_on_file(synthax_error_json) + assert exc_info.value.args[0].startswith(f"Can't format {synthax_error_json} because it has a synthax error.") + + +def test_run_prettier_pre_commit_on_synthax_error_file(synthax_error_json, ensure_prettier_is_not_found): + with pytest.raises(ValueError) as exc_info: + nf_core.lint_utils.run_prettier_on_file(synthax_error_json) + assert exc_info.value.args[0].startswith(f"Can't format {synthax_error_json} because it has a synthax error.") From 6a8af3d6dc357b7a10baaf5c519d8d8801b25460 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 13:03:20 +0100 Subject: [PATCH 523/854] fix typo and modules.json update when new component type --- nf_core/__main__.py | 4 ++-- nf_core/modules/modules_json.py | 30 +++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a42776d685..3d1c5cba91 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -557,7 +557,7 @@ def install(ctx, tool, dir, prompt, force, sha): @click.option( "-r", "--recursive", - is_flat=True, + is_flag=True, default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) @@ -1097,7 +1097,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin @click.option( "-r", "--recursive", - is_flat=True, + is_flag=True, default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index f636ac562d..987045f43c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1042,11 +1042,27 @@ def components_with_repos(): if install_dir in install_directories ][0] repo_entry = self.determine_branches_and_shas(component_type, install_dir, remote_url, components) - if remote_url in self.modules_json["repos"]: + try: self.modules_json["repos"][remote_url][component_type][install_dir].update(repo_entry) - else: - self.modules_json["repos"][remote_url] = { - component_type: { - install_dir: repo_entry, - } - } + except KeyError: + try: + self.modules_json["repos"][remote_url][component_type].update({install_dir: repo_entry}) + except KeyError: + try: + self.modules_json["repos"][remote_url].update( + { + component_type: { + install_dir: repo_entry, + } + } + ) + except KeyError: + self.modules_json["repos"].update( + { + remote_url: { + component_type: { + install_dir: repo_entry, + } + } + } + ) From 7f69ac349d410797c9900740a47bb760f0ab1463 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 13:09:42 +0100 Subject: [PATCH 524/854] remove duplicated sw install objects --- tests/subworkflows/install.py | 32 ++++++++++++++++---------------- tests/test_subworkflows.py | 6 ------ 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 94b2d79ba0..f394ef4066 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -16,35 +16,35 @@ def test_subworkflow_install_nopipeline(self): """Test installing a subworkflow - no pipeline given""" - self.subworkflow_install.dir = None - assert self.subworkflow_install.install("foo") is False + self.sw_install.dir = None + assert self.sw_install.install("foo") is False @with_temporary_folder def test_subworkflows_install_emptypipeline(self, tmpdir): """Test installing a subworkflow - empty dir given""" os.mkdir(os.path.join(tmpdir, "nf-core-pipe")) - self.subworkflow_install.dir = os.path.join(tmpdir, "nf-core-pipe") + self.sw_install.dir = os.path.join(tmpdir, "nf-core-pipe") with pytest.raises(UserWarning) as excinfo: - self.subworkflow_install.install("foo") + self.sw_install.install("foo") assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) def test_subworkflows_install_nosubworkflow(self): """Test installing a subworkflow - unrecognised subworkflow given""" - assert self.subworkflow_install.install("foo") is False + assert self.sw_install.install("foo") is False def test_subworkflows_install_bam_sort_stats_samtools(self): """Test installing a subworkflow - bam_sort_stats_samtools""" - assert self.subworkflow_install.install("bam_sort_stats_samtools") is not False - subworkflow_path = os.path.join(self.subworkflow_install.dir, "subworkflows", "nf-core", "bam_sort_stats_samtools") - sub_subworkflow_path = os.path.join(self.subworkflow_install.dir, "subworkflows", "nf-core", "bam_stats_samtools") - samtools_index_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") - samtools_sort_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "sort") - samtools_stats_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "stats") - samtools_idxstats_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "idxstats") - samtools_flagstat_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "flagstat") + assert self.sw_install.install("bam_sort_stats_samtools") is not False + subworkflow_path = os.path.join(self.sw_install.dir, "subworkflows", "nf-core", "bam_sort_stats_samtools") + sub_subworkflow_path = os.path.join(self.sw_install.dir, "subworkflows", "nf-core", "bam_stats_samtools") + samtools_index_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "index") + samtools_sort_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "sort") + samtools_stats_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "stats") + samtools_idxstats_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "idxstats") + samtools_flagstat_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "flagstat") assert os.path.exists(subworkflow_path) assert os.path.exists(sub_subworkflow_path) assert os.path.exists(samtools_index_path) @@ -56,13 +56,13 @@ def test_subworkflows_install_bam_sort_stats_samtools(self): def test_subworkflows_install_bam_sort_stats_samtools_twice(self): """Test installing a subworkflow - bam_sort_stats_samtools already there""" - self.subworkflow_install.install("bam_sort_stats_samtools") - assert self.subworkflow_install.install("bam_sort_stats_samtools") is False + self.sw_install.install("bam_sort_stats_samtools") + assert self.sw_install.install("bam_sort_stats_samtools") is False def test_subworkflows_install_from_gitlab(self): """Test installing a subworkflow from GitLab""" - assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True + assert self.sw_install_gitlab.install("bam_stats_samtools") is True # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) assert ( diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index be15e2f15b..5f2b0440a3 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -52,12 +52,6 @@ def setUp(self): "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() - # Set up install objects - self.subworkflow_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=False) - self.subworkflow_install_gitlab = nf_core.subworkflows.SubworkflowInstall( - self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH - ) - # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) From 5f4711e1f5819ae6f249aaef5b55f4b2c4b3851e Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 14 Nov 2022 14:07:29 +0100 Subject: [PATCH 525/854] remove unused imports --- tests/test_lint_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_lint_utils.py b/tests/test_lint_utils.py index 1d864d77df..88d5553346 100644 --- a/tests/test_lint_utils.py +++ b/tests/test_lint_utils.py @@ -1,6 +1,4 @@ import shutil -from pathlib import Path -from unittest import mock import git import pytest From 4fd9b8cef09e46d10689a153561d0fc751790fd5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 15:55:33 +0100 Subject: [PATCH 526/854] add tests for subworkflows update --- nf_core/components/update.py | 5 +- tests/subworkflows/update.py | 385 ++++++++++++++++++----------------- tests/test_subworkflows.py | 25 ++- tests/utils.py | 2 + 4 files changed, 227 insertions(+), 190 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 4648a31875..178f2eb25c 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -113,6 +113,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr return False # Get the list of modules/subworkflows to update, and their version information + print(self.update_all) components_info = ( self.get_all_components_info() if self.update_all else [self.get_single_component_info(component)] ) @@ -709,7 +710,9 @@ def move_files_from_tmp_dir(self, component, install_folder, repo_path, new_vers log.info(f"Updating '{repo_path}/{component}'") log.debug(f"Updating {self.component_type[:-1]} '{component}' to {new_version} from {repo_path}") - def try_apply_patch(self, component, repo_path, patch_relpath, component_dir, component_install_dir, write_file=True): + def try_apply_patch( + self, component, repo_path, patch_relpath, component_dir, component_install_dir, write_file=True + ): """ Try applying a patch file to the new module/subworkflow files diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 60f869ae4c..182ed00d19 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -1,296 +1,305 @@ import filecmp -import os import shutil import tempfile +from pathlib import Path import yaml import nf_core.utils -from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.update import ModuleUpdate +from nf_core.subworkflows.update import SubworkflowUpdate -from ..utils import ( - GITLAB_BRANCH_TEST_BRANCH, - GITLAB_BRANCH_TEST_NEW_SHA, - GITLAB_BRANCH_TEST_OLD_SHA, - GITLAB_DEFAULT_BRANCH, - GITLAB_REPO, - GITLAB_URL, - OLD_TRIMGALORE_BRANCH, - OLD_TRIMGALORE_SHA, -) +from ..utils import OLD_SUBWORKFLOWS_SHA def test_install_and_update(self): - """Installs a module in the pipeline and updates it (no change)""" - self.mods_install.install("trimgalore") - update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) + """Installs a subworkflow in the pipeline and updates it (no change)""" + self.sw_install.install("bam_stats_samtools") + update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False) - # Copy the module files and check that they are unaffected by the update + # Copy the sw files and check that they are unaffected by the update tmpdir = tempfile.mkdtemp() - trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "trimgalore") - shutil.copytree(trimgalore_path, trimgalore_tmpdir) + shutil.rmtree(tmpdir) + sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "bam_stats_samtools") + shutil.copytree(sw_path, tmpdir) - assert update_obj.update("trimgalore") is True - assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + assert update_obj.update("bam_stats_samtools") is True + assert cmp_component(tmpdir, sw_path) is True def test_install_at_hash_and_update(self): - """Installs an old version of a module in the pipeline and updates it""" - assert self.mods_install_old.install("trimgalore") - update_obj = ModuleUpdate( - self.pipeline_dir, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH - ) + """Installs an old version of a subworkflow in the pipeline and updates it""" + assert self.sw_install_old.install("fastq_align_bowtie2") + update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False, recursive=True) + old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + print(old_mod_json) - # Copy the module files and check that they are affected by the update + # Copy the sw files and check that they are affected by the update tmpdir = tempfile.mkdtemp() - trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") - shutil.copytree(trimgalore_path, trimgalore_tmpdir) + shutil.rmtree(tmpdir) + sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "fastq_align_bowtie2") + shutil.copytree(sw_path, tmpdir) - assert update_obj.update("trimgalore") is True - assert cmp_module(trimgalore_tmpdir, trimgalore_path) is False + assert update_obj.update("fastq_align_bowtie2") is True + assert cmp_component(tmpdir, sw_path) is False # Check that the modules.json is correctly updated - mod_json_obj = ModulesJson(self.pipeline_dir) - mod_json = mod_json_obj.get_modules_json() - # Get the up-to-date git_sha for the module from the ModulesRepo object - correct_git_sha = update_obj.modules_repo.get_latest_component_version("trimgalore", "modules") - current_git_sha = mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] - assert correct_git_sha == current_git_sha + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + print(mod_json) + # Get the up-to-date git_sha for the sw from the ModulesRepo object + assert ( + old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ + "git_sha" + ] + != mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ + "git_sha" + ] + ) def test_install_at_hash_and_update_and_save_diff_to_file(self): - """Installs an old version of a module in the pipeline and updates it""" - self.mods_install_old.install("trimgalore") - patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") - update_obj = ModuleUpdate( - self.pipeline_dir, - save_diff_fn=patch_path, - sha=OLD_TRIMGALORE_SHA, - remote_url=GITLAB_URL, - branch=OLD_TRIMGALORE_BRANCH, - ) + """Installs an old version of a sw in the pipeline and updates it. Save differences to a file.""" + assert self.sw_install_old.install("fastq_align_bowtie2") + patch_path = Path(self.pipeline_dir, "fastq_align_bowtie2.patch") + update_obj = SubworkflowUpdate(self.pipeline_dir, save_diff_fn=patch_path, recursive=True) - # Copy the module files and check that they are affected by the update + # Copy the sw files and check that they are affected by the update tmpdir = tempfile.mkdtemp() - trimgalore_tmpdir = os.path.join(tmpdir, "trimgalore") - trimgalore_path = os.path.join(self.pipeline_dir, "modules", GITLAB_REPO, "trimgalore") - shutil.copytree(trimgalore_path, trimgalore_tmpdir) + shutil.rmtree(tmpdir) + sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "fastq_align_bowtie2") + shutil.copytree(sw_path, tmpdir) - assert update_obj.update("trimgalore") is True - assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + assert update_obj.update("fastq_align_bowtie2") is True + assert cmp_component(tmpdir, sw_path) is True - # TODO: Apply the patch to the module + with open(patch_path, "r") as fh: + line = fh.readline() + assert line.startswith( + "Changes in module 'nf-core/fastq_align_bowtie2' between (f3c078809a2513f1c95de14f6633fe1f03572fdb) and" + ) def test_update_all(self): - """Updates all modules present in the pipeline""" - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) - # Get the current modules.json + """Updates all subworkflows present in the pipeline""" + # Install subworkflows fastq_align_bowtie2, bam_sort_stats_samtools, bam_stats_samtools + self.sw_install.install("fastq_align_bowtie2") + # Update all subworkflows + update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=True, show_diff=False) assert update_obj.update() is True # We must reload the modules.json to get the updated version mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() - # Loop through all modules and check that they are updated (according to the modules.json file) - for mod in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]: - correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] - current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + print(mod_json) + # Loop through all subworkflows and check that they are updated (according to the modules.json file) + for sw in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]: + correct_git_sha = list(update_obj.modules_repo.get_component_git_log(sw, "subworkflows", depth=1))[0]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME][sw]["git_sha"] assert correct_git_sha == current_git_sha def test_update_with_config_fixed_version(self): """Try updating when there are entries in the .nf-core.yml""" - # Install trimgalore at the latest version - assert self.mods_install_trimgalore.install("trimgalore") + # Install subworkflow at the latest version + assert self.sw_install.install("fastq_align_bowtie2") - # Fix the trimgalore version in the .nf-core.yml to an old version - update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": OLD_TRIMGALORE_SHA}}} + # Fix the subworkflow version in the .nf-core.yml to an old version + update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"fastq_align_bowtie2": OLD_SUBWORKFLOWS_SHA}}} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + with open(Path(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) - # Update all modules in the pipeline - update_obj = ModuleUpdate( - self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH - ) + # Update all subworkflows in the pipeline + update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=True, show_diff=False) assert update_obj.update() is True - # Check that the git sha for trimgalore is correctly downgraded + # Check that the git sha for fastq_align_bowtie2 is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] - assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] - assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + assert "fastq_align_bowtie2" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME] + assert ( + "git_sha" + in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"] + ) + assert ( + mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ + "git_sha" + ] + == OLD_SUBWORKFLOWS_SHA + ) def test_update_with_config_dont_update(self): - """Try updating when module is to be ignored""" - # Install an old version of trimgalore - self.mods_install_old.install("trimgalore") + """Try updating when sw is to be ignored""" + # Install an old version of fastq_align_bowtie2 + self.sw_install_old.install("fastq_align_bowtie2") - # Set the trimgalore field to no update in the .nf-core.yml - update_config = {GITLAB_URL: {GITLAB_REPO: {"trimgalore": False}}} + # Set the fastq_align_bowtie2 field to no update in the .nf-core.yml + update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"fastq_align_bowtie2": False}}} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + with open(Path(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) # Update all modules in the pipeline - update_obj = ModuleUpdate( - self.pipeline_dir, - update_all=True, - show_diff=False, - sha=OLD_TRIMGALORE_SHA, - remote_url=GITLAB_URL, - branch=OLD_TRIMGALORE_BRANCH, - ) + update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=True, show_diff=False) assert update_obj.update() is True - # Check that the git sha for trimgalore is correctly downgraded + # Check that the git sha for fastq_align_bowtie2 is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "trimgalore" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO] - assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] - assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + assert "fastq_align_bowtie2" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME] + assert ( + "git_sha" + in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"] + ) + assert ( + mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ + "git_sha" + ] + == OLD_SUBWORKFLOWS_SHA + ) def test_update_with_config_fix_all(self): - """Fix the version of all nf-core modules""" - self.mods_install_trimgalore.install("trimgalore") + """Fix the version of all nf-core subworkflows""" + # Install subworkflow at the latest version + assert self.sw_install.install("fastq_align_bowtie2") - # Fix the version of all nf-core modules in the .nf-core.yml to an old version - update_config = {GITLAB_URL: OLD_TRIMGALORE_SHA} + # Fix the version of all nf-core subworkflows in the .nf-core.yml to an old version + update_config = {NF_CORE_MODULES_REMOTE: OLD_SUBWORKFLOWS_SHA} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + with open(Path(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) - # Update all modules in the pipeline - update_obj = ModuleUpdate( - self.pipeline_dir, update_all=True, show_diff=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH - ) + # Update all subworkflows in the pipeline + update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=True, show_diff=False) assert update_obj.update() is True - # Check that the git sha for trimgalore is correctly downgraded + # Check that the git sha for fastq_align_bowtie2 is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"] - assert mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]["trimgalore"]["git_sha"] == OLD_TRIMGALORE_SHA + assert ( + "git_sha" + in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"] + ) + assert ( + mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ + "git_sha" + ] + == OLD_SUBWORKFLOWS_SHA + ) def test_update_with_config_no_updates(self): - """Don't update any nf-core modules""" - self.mods_install_old.install("trimgalore") + """Don't update any nf-core subworkflows""" + # Install an old version of fastq_align_bowtie2 + self.sw_install_old.install("fastq_align_bowtie2") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - # Fix the version of all nf-core modules in the .nf-core.yml to an old version - update_config = {GITLAB_URL: False} + # Set all repository updates to False + update_config = {NF_CORE_MODULES_REMOTE: False} tools_config = nf_core.utils.load_tools_config(self.pipeline_dir) tools_config["update"] = update_config - with open(os.path.join(self.pipeline_dir, ".nf-core.yml"), "w") as f: + with open(Path(self.pipeline_dir, ".nf-core.yml"), "w") as f: yaml.dump(tools_config, f) - # Update all modules in the pipeline - update_obj = ModuleUpdate( - self.pipeline_dir, - update_all=True, - show_diff=False, - sha=OLD_TRIMGALORE_SHA, - remote_url=GITLAB_URL, - branch=OLD_TRIMGALORE_BRANCH, - ) + # Update all subworkflows in the pipeline + update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=True, show_diff=False) assert update_obj.update() is True - # Check that the git sha for trimgalore is correctly downgraded and none of the modules has changed + # Check that the git sha for fastq_align_bowtie2 is correctly downgraded and none of the subworkflows has changed mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - for module in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO]: - assert "git_sha" in mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module] + for sw in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]: + assert "git_sha" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME][sw] assert ( - mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module]["git_sha"] - == old_mod_json["repos"][GITLAB_URL]["modules"][GITLAB_REPO][module]["git_sha"] + mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME][sw]["git_sha"] + == old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME][sw]["git_sha"] ) -def test_update_different_branch_single_module(self): - """Try updating a module in a specific branch""" - install_obj = ModuleInstall( - self.pipeline_dir, - prompt=False, - force=False, - remote_url=GITLAB_URL, - branch=GITLAB_BRANCH_TEST_BRANCH, - sha=GITLAB_BRANCH_TEST_OLD_SHA, - ) - assert install_obj.install("fastp") +def test_update_all_linked_components_from_subworkflow(self): + """Update a subworkflow and all modules and subworkflows used on it""" + # Install an old version of fastq_align_bowtie2 + self.sw_install_old.install("fastq_align_bowtie2") + old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - update_obj = ModuleUpdate( - self.pipeline_dir, recursive=True, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False - ) - update_obj.update("fastp") + # Copy the sw files and check that they are affected by the update + tmpdir = tempfile.mkdtemp() + shutil.rmtree(tmpdir) + subworkflows_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME) + modules_path = Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME) + shutil.copytree(subworkflows_path, Path(tmpdir, "subworkflows")) + shutil.copytree(modules_path, Path(tmpdir, "modules")) + + # Update fastq_align_bowtie2 and all modules and subworkflows used by that + update_obj = SubworkflowUpdate(self.pipeline_dir, recursive=True, show_diff=False) + assert update_obj.update("fastq_align_bowtie2") is True - # Verify that the branch entry was updated correctly - modules_json = ModulesJson(self.pipeline_dir) + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + # Loop through all modules and subworkflows used in fastq_align_bowtie2 + # check that they are updated (according to the modules.json file) + for sw in ["fastq_align_bowtie2", "bam_sort_stats_samtools", "bam_stats_samtools"]: + assert ( + old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME][sw]["git_sha"] + != mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME][sw]["git_sha"] + ) + for mod in [ + "bowtie2/align", + "samtools/index", + "samtools/sort", + "samtools/flagstat", + "samtools/idxstats", + "samtools/stats", + ]: + assert ( + old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + != mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + ) + # Check that the subworkflow files are updated assert ( - modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) - == GITLAB_BRANCH_TEST_BRANCH + cmp_component( + Path(tmpdir, "subworkflows", "fastq_align_bowtie2"), Path(subworkflows_path, "fastq_align_bowtie2") + ) + is False ) - assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA -def test_update_different_branch_mixed_modules_main(self): - """Try updating all modules where MultiQC is installed from main branch""" - # Install fastp - assert self.mods_install_gitlab_old.install("fastp") +def test_update_all_subworkflows_from_module(self): + """Update a module and all subworkflows that use this module""" + # Install an old version of fastq_align_bowtie2 and thus all modules used by it (bowtie2/align) + self.sw_install_old.install("fastq_align_bowtie2") + old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - # Install MultiQC from gitlab default branch - assert self.mods_install_gitlab.install("multiqc") + # Copy the sw files and check that they are affected by the update + tmpdir = tempfile.mkdtemp() + shutil.rmtree(tmpdir) + sw_path = Path(self.pipeline_dir, "subworkflows", NF_CORE_MODULES_NAME, "fastq_align_bowtie2") + shutil.copytree(sw_path, Path(tmpdir, "fastq_align_bowtie2")) - # Try updating - update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=False) - assert update_obj.update() is True + # Update bowtie2/align and all subworkflows using it + update_obj = ModuleUpdate(self.pipeline_dir, recursive=True, show_diff=False) + assert update_obj.update("bowtie2/align") is True - modules_json = ModulesJson(self.pipeline_dir) - # Verify that the branch entry was updated correctly - assert ( - modules_json.get_component_branch(self.component_type, "fastp", GITLAB_URL, GITLAB_REPO) - == GITLAB_BRANCH_TEST_BRANCH - ) - assert modules_json.get_module_version("fastp", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA - # MultiQC is present in both branches but should've been updated using the 'main' branch + mod_json = ModulesJson(self.pipeline_dir).get_modules_json() + # Check that bowtie2/align and fastq_align_bowtie2 are updated assert ( - modules_json.get_component_branch(self.component_type, "multiqc", GITLAB_URL, GITLAB_REPO) - == GITLAB_DEFAULT_BRANCH - ) - - -def test_update_different_branch_mix_modules_branch_test(self): - """Try updating all modules where MultiQC is installed from branch-test branch""" - # Install multiqc from the branch-test branch - assert self.mods_install_gitlab_old.install( - "multiqc" - ) # Force as the same module is installed from github nf-core modules repo - modules_json = ModulesJson(self.pipeline_dir) - update_obj = ModuleUpdate( - self.pipeline_dir, - update_all=True, - show_diff=False, - remote_url=GITLAB_URL, - branch=GITLAB_BRANCH_TEST_BRANCH, - sha=GITLAB_BRANCH_TEST_NEW_SHA, + old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ + "git_sha" + ] + != mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ + "git_sha" + ] ) - assert update_obj.update() - + assert cmp_component(Path(tmpdir, "fastq_align_bowtie2"), sw_path) is False assert ( - modules_json.get_component_branch(self.component_type, "multiqc", GITLAB_URL, GITLAB_REPO) - == GITLAB_BRANCH_TEST_BRANCH + old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["bowtie2/align"]["git_sha"] + != mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]["bowtie2/align"]["git_sha"] ) - assert modules_json.get_module_version("multiqc", GITLAB_URL, GITLAB_REPO) == GITLAB_BRANCH_TEST_NEW_SHA -def cmp_module(dir1, dir2): - """Compare two versions of the same module""" +def cmp_component(dir1, dir2): + """Compare two versions of the same component""" files = ["main.nf", "meta.yml"] - return all(filecmp.cmp(os.path.join(dir1, f), os.path.join(dir2, f), shallow=False) for f in files) + return all(filecmp.cmp(Path(dir1, f), Path(dir2, f), shallow=False) for f in files) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 5f2b0440a3..abdd550ecb 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -11,7 +11,12 @@ import nf_core.modules import nf_core.subworkflows -from .utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, mock_api_calls +from .utils import ( + GITLAB_SUBWORKFLOWS_BRANCH, + GITLAB_URL, + OLD_SUBWORKFLOWS_SHA, + mock_api_calls, +) def create_modules_repo_dummy(tmp_dir): @@ -60,6 +65,12 @@ def setUp(self): self.sw_install_gitlab = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH ) + self.sw_install_old = nf_core.subworkflows.SubworkflowInstall( + self.pipeline_dir, + prompt=False, + force=False, + sha=OLD_SUBWORKFLOWS_SHA, + ) ############################################ # Test of the individual modules commands. # @@ -90,3 +101,15 @@ def setUp(self): test_subworkflows_test_no_installed_subworkflows, test_subworkflows_test_no_name_no_prompts, ) + from .subworkflows.update import ( + test_install_and_update, + test_install_at_hash_and_update, + test_install_at_hash_and_update_and_save_diff_to_file, + test_update_all, + test_update_all_linked_components_from_subworkflow, + test_update_all_subworkflows_from_module, + test_update_with_config_dont_update, + test_update_with_config_fix_all, + test_update_with_config_fixed_version, + test_update_with_config_no_updates, + ) diff --git a/tests/utils.py b/tests/utils.py index 65c7d48758..abe1626496 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,12 +16,14 @@ GITLAB_REPO = "nf-core" GITLAB_DEFAULT_BRANCH = "main-restructure" GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows" +OLD_SUBWORKFLOWS_SHA = "f3c078809a2513f1c95de14f6633fe1f03572fdb" # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester-restructure" GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" GITLAB_BRANCH_TEST_NEW_SHA = "2f5f180f6e705bb81d6e7742dc2f24bf4a0c721e" + def with_temporary_folder(func): """ Call the decorated funtion under the tempfile.TemporaryDirectory From 30f2c07c8d9ef4c7f4e24ee1e0e30f7cb6c1aeed Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 16:35:11 +0100 Subject: [PATCH 527/854] all subworkflows tests pass --- nf_core/components/update.py | 43 +++++++++++++++++++----------------- tests/modules/update.py | 1 - tests/subworkflows/update.py | 3 --- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 178f2eb25c..d59f94610e 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -113,7 +113,6 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr return False # Get the list of modules/subworkflows to update, and their version information - print(self.update_all) components_info = ( self.get_all_components_info() if self.update_all else [self.get_single_component_info(component)] ) @@ -364,28 +363,32 @@ def get_single_component_info(self, component): ) sha = self.sha - if component in self.update_config.get(self.modules_repo.remote_url, {}).get(install_dir, {}): + config_entry = None + if isinstance(self.update_config.get(self.modules_repo.remote_url, {}), str): + # If the repo entry is a string, it's the sha to update to + config_entry = self.update_config.get(self.modules_repo.remote_url, {}) + elif component in self.update_config.get(self.modules_repo.remote_url, {}).get(install_dir, {}): # If the component to update is in .nf-core.yml config file config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(component) - if config_entry is not None and config_entry is not True: - if config_entry is False: - raise UserWarning( - f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is set to False" - ) - if not isinstance(config_entry, str): - raise UserWarning( - f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is of wrong type" - ) + if config_entry is not None and config_entry is not True: + if config_entry is False: + raise UserWarning( + f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is set to False" + ) + if not isinstance(config_entry, str): + raise UserWarning( + f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is of wrong type" + ) - sha = config_entry - if self.sha is not None: - log.warning( - f"Found entry in '.nf-core.yml' for {self.component_type[:-1]} '{component}' " - "which will override version specified with '--sha'" - ) - else: - log.info(f"Found entry in '.nf-core.yml' for {self.component_type[:-1]} '{component}'") - log.info(f"Updating component to ({sha})") + sha = config_entry + if self.sha is not None: + log.warning( + f"Found entry in '.nf-core.yml' for {self.component_type[:-1]} '{component}' " + "which will override version specified with '--sha'" + ) + else: + log.info(f"Found entry in '.nf-core.yml' for {self.component_type[:-1]} '{component}'") + log.info(f"Updating component to ({sha})") # Check if the update branch is the same as the installation branch current_branch = self.modules_json.get_component_branch( diff --git a/tests/modules/update.py b/tests/modules/update.py index 2620aa5930..a4121f917c 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -357,7 +357,6 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): for mod in ["custom/dumpsoftwareversions", "fastqc"]: correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] - print(correct_git_sha, current_git_sha) assert correct_git_sha != current_git_sha diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 182ed00d19..0b061e0acf 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -34,7 +34,6 @@ def test_install_at_hash_and_update(self): assert self.sw_install_old.install("fastq_align_bowtie2") update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False, recursive=True) old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - print(old_mod_json) # Copy the sw files and check that they are affected by the update tmpdir = tempfile.mkdtemp() @@ -47,7 +46,6 @@ def test_install_at_hash_and_update(self): # Check that the modules.json is correctly updated mod_json = ModulesJson(self.pipeline_dir).get_modules_json() - print(mod_json) # Get the up-to-date git_sha for the sw from the ModulesRepo object assert ( old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]["fastq_align_bowtie2"][ @@ -92,7 +90,6 @@ def test_update_all(self): # We must reload the modules.json to get the updated version mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() - print(mod_json) # Loop through all subworkflows and check that they are updated (according to the modules.json file) for sw in mod_json["repos"][NF_CORE_MODULES_REMOTE]["subworkflows"][NF_CORE_MODULES_NAME]: correct_git_sha = list(update_obj.modules_repo.get_component_git_log(sw, "subworkflows", depth=1))[0]["git_sha"] From 3e6660e957e4f5930df9482b024f5d8054fdddd7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 14 Nov 2022 16:41:40 +0100 Subject: [PATCH 528/854] run black --- tests/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index abe1626496..07143171fd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -23,7 +23,6 @@ GITLAB_BRANCH_TEST_NEW_SHA = "2f5f180f6e705bb81d6e7742dc2f24bf4a0c721e" - def with_temporary_folder(func): """ Call the decorated funtion under the tempfile.TemporaryDirectory From c5421e51e5ae66221696fa829b62e99a26e759f9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Nov 2022 17:40:24 +0100 Subject: [PATCH 529/854] recursively remove entries --- nf_core/components/remove.py | 30 ++++++++++---- nf_core/modules/modules_json.py | 70 +++++++++++++++++++++++++++------ tests/subworkflows/remove.py | 18 ++++++--- 3 files changed, 94 insertions(+), 24 deletions(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 74476aeefc..b717ad599e 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -55,14 +55,30 @@ def remove(self, component): modules_json.remove_entry(self.component_type, component, self.modules_repo.remote_url, repo_path) return False + removed_by = None + dependent_components = {component: self.component_type} + if self.component_type == "subworkflows": + removed_by = component + if removed_by is not None: + dependent_components.update( + modules_json.get_dependent_components( + self.component_type, component, self.modules_repo.remote_url, repo_path, dependent_components + ) + ) + # remove all dependent components based on installed_by entry # Remove entry from modules.json - removed_by = component if self.component_type == "subworkflows" else None removed = False - removed = modules_json.remove_entry( - self.component_type, component, self.modules_repo.remote_url, repo_path, removed_by=removed_by - ) - # Remove the module files - if removed: - removed = self.clear_component_dir(component, module_dir) + for component_name, component_type in dependent_components.items(): + removed_component = modules_json.remove_entry( + component_type, + component_name, + self.modules_repo.remote_url, + repo_path, + removed_by=removed_by, + ) + # Remove the component files if the entry was removed from modules.json + if removed_component: + component_dir = Path(self.dir, component_type, repo_path, component_name) + removed = True if self.clear_component_dir(component, component_dir) or removed else False return removed diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 25e52b36ca..7438526560 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -675,15 +675,16 @@ def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=N Removes an entry from the 'modules.json' file. Args: - component_type (Str): Type of component [modules, subworkflows] - name (str): Name of the module to be removed - repo_url (str): URL of the repository containing the module - install_dir (str): Name of the directory where modules are installed + component_type (str): Type of component [modules, subworkflows] + name (str): Name of the component to be removed + repo_url (str): URL of the repository containing the component + install_dir (str): Name of the directory where components are installed + removed_by (str): Name of the component that wants to remove the component Returns: - (bool): True if the entry has actually been removed from the modules.json, False otherwise + (bool): return True if the component was removed, False if it was not found or is still depended on """ - if removed_by is None: + if removed_by is None or removed_by == name: removed_by = component_type if not self.modules_json: return False @@ -693,25 +694,28 @@ def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=N if removed_by in repo_entry[component_type][install_dir][name]["installed_by"]: repo_entry[component_type][install_dir][name]["installed_by"].remove(removed_by) if len(repo_entry[component_type][install_dir][name]["installed_by"]) == 0: - repo_entry[component_type][install_dir].pop(name) + self.modules_json["repos"][repo_url][component_type][install_dir].pop(name) + self.dump() + if len(repo_entry[component_type][install_dir]) == 0: + + self.modules_json["repos"][repo_url].pop(component_type) return True + self.dump() + return False else: log.error( f"Could not find 'installed_by' entry for '{removed_by}' in 'modules.json' file. Did you install it first?" ) - else: log.warning( f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file." ) return False - if len(repo_entry[component_type][install_dir]) == 0: - self.modules_json["repos"].pop(repo_url) + else: log.warning(f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file.") return False - self.dump() return False def add_patch_entry(self, module_name, repo_url, install_dir, patch_filename, write_file=True): @@ -911,6 +915,50 @@ def get_all_components(self, component_type): return self.pipeline_components + def get_dependent_components( + self, + component_type, + name, + repo_url, + install_dir, + dependent_components, + ): + """ + Retrieves all pipeline modules/subworkflows that are reported in the modules.json + as being installed by the given component + + Args: + component_type (str): Type of component [modules, subworkflows] + name (str): Name of the component to find dependencies for + repo_url (str): URL of the repository containing the components + install_dir (str): Name of the directory where components are installed + + Returns: + (dict[str: str,]): Dictionary indexed with the component names, with component_type as value + """ + + if self.modules_json is None: + self.load() + component_types = ["modules"] if component_type == "modules" else ["modules", "subworkflows"] + # Find all components that have an entry of install by of a given component, recursively call this function for subworkflows + for type in component_types: + components = self.modules_json["repos"][repo_url][type][install_dir].items() + for component_name, component_entry in components: + if name in component_entry["installed_by"]: + dependent_components[component_name] = type + if type == "subworkflows": + dependent_components.update( + self.get_dependent_components( + type, + component_name, + repo_url, + install_dir, + dependent_components, + ) + ) + + return dependent_components + def get_component_branch(self, component_type, component, repo_url, install_dir): """ Gets the branch from which the module/subworkflow was installed diff --git a/tests/subworkflows/remove.py b/tests/subworkflows/remove.py index 28100c5e52..936d164383 100644 --- a/tests/subworkflows/remove.py +++ b/tests/subworkflows/remove.py @@ -1,4 +1,5 @@ from pathlib import Path +from nf_core.modules.modules_json import ModulesJson def test_subworkflows_remove_uninstalled_subworkflow(self): @@ -10,12 +11,17 @@ def test_subworkflows_remove_subworkflow(self): """Test removing subworkflow and all it's dependencies after installing it""" self.subworkflow_install.install("bam_sort_stats_samtools") - subworkflow_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "subworkflows") + subworkflow_path = Path(self.subworkflow_install.dir, "subworkflows", "nf-core") bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools") bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") - samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "modules", "samtools", "index") - + samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_before = mod_json_obj.get_modules_json() assert self.subworkflow_remove.remove("bam_sort_stats_samtools") + mod_json_after = mod_json_obj.get_modules_json() + import ipdb + + ipdb.set_trace() assert Path.exists(subworkflow_path) is False assert Path.exists(bam_sort_stats_samtools_path) is False assert Path.exists(bam_stats_samtools_path) is False @@ -27,14 +33,14 @@ def test_subworkflows_remove_one_of_two_subworkflow(self): self.subworkflow_install.install("bam_sort_stats_samtools") self.subworkflow_install.install("bam_stats_samtools") - subworkflow_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "subworkflows") + subworkflow_path = Path(self.subworkflow_install.dir, "subworkflows", "nf-core") bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools") bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") - samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "modules", "samtools", "index") + samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") assert self.subworkflow_remove.remove("bam_sort_stats_samtools") - assert Path.exists(subworkflow_path) is False + assert Path.exists(subworkflow_path) is True assert Path.exists(bam_sort_stats_samtools_path) is False assert Path.exists(bam_stats_samtools_path) is True assert Path.exists(samtools_index_path) is True From 62af958a70082a425facacb6c6876cdbaa580be8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 14 Nov 2022 17:40:58 +0100 Subject: [PATCH 530/854] rewrite remove_dir command to better handle empty directories --- nf_core/components/components_command.py | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index d671131c16..9a7511ca49 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -84,23 +84,31 @@ def has_modules_file(self): ModulesJson(self.dir).create() def clear_component_dir(self, component_name, component_dir): - """Removes all files in the module/subworkflow directory""" + """ + Removes all files in the module/subworkflow directory + + Args: + component_name (str): The name of the module/subworkflow + component_dir (str): The path to the module/subworkflow in the module repository + + """ + try: shutil.rmtree(component_dir) - if self.component_type == "modules": - # Try cleaning up empty parent if tool/subtool and tool/ is empty - if component_name.count("/") > 0: - parent_dir = os.path.dirname(component_dir) + # remove all empty directories + for dir_path, dir_names, filenames in os.walk(self.dir, topdown=False): + if not dir_names and not filenames: try: - os.rmdir(parent_dir) + os.rmdir(dir_path) except OSError: - log.debug(f"Parent directory not empty: '{parent_dir}'") + pass else: - log.debug(f"Deleted orphan tool directory: '{parent_dir}'") - log.debug(f"Successfully removed {component_name} {self.component_type[:-1]}") + log.debug(f"Deleted directory: '{dir_path}'") + + log.debug(f"Successfully removed {self.component_type[:-1]} {component_name}") return True except OSError as e: - log.error(f"Could not remove {self.component_type[:-1]}: {e}") + log.error(f"Could not remove {self.component_type[:-1]} {component_name}: {e}") return False def components_from_repo(self, install_dir): From 9a1cfdcb20b12dbd08e30c5a9e14aaefae0db6a7 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 14 Nov 2022 17:47:26 +0100 Subject: [PATCH 531/854] adherer to prettier formatting --- nf_core/module-template/modules/meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 23c8b06da8..c95e3e1d84 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -21,7 +21,7 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - # {% endif %} + #{% endif %} ## TODO nf-core: Delete / customise this example input - bam: type: file @@ -36,7 +36,7 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - # {% endif %} + #{% endif %} - versions: type: file description: File containing software versions From 4fbd447601a1ff45a98dcf7126b382acb966e41e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Nov 2022 11:45:34 +0100 Subject: [PATCH 532/854] fix tests --- nf_core/components/components_utils.py | 2 +- nf_core/subworkflows/install.py | 2 +- tests/subworkflows/remove.py | 8 +++----- tests/test_modules.py | 2 ++ tests/test_subworkflows.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index bbe629adfc..7fd049f6fc 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -87,7 +87,7 @@ def prompt_component_version_sha(component_name, component_type, modules_repo, i git_sha = "" page_nbr = 1 - all_commits = modules_repo.get_component_git_log(component_name, subworkflow) + all_commits = modules_repo.get_component_git_log(component_name, component_type) next_page_commits = [next(all_commits, None) for _ in range(10)] next_page_commits = [commit for commit in next_page_commits if commit is not None] diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index 2df0c48970..3553f48b0e 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -82,7 +82,7 @@ def install(self, subworkflow, silent=False): f"Subworkflow is already installed and force is not set.\nAdding the new installation source {self.installed_by} for subworkflow {subworkflow} to 'modules.json' without installing the subworkflow." ) modules_json.load() - modules_json.update(self.modules_repo, subworkflow, current_version, self.installed_by) + modules_json.update_subworkflow(self.modules_repo, subworkflow, current_version, self.installed_by) return False version = nf_core.components.components_install.get_version( diff --git a/tests/subworkflows/remove.py b/tests/subworkflows/remove.py index 936d164383..65be9c3d22 100644 --- a/tests/subworkflows/remove.py +++ b/tests/subworkflows/remove.py @@ -19,9 +19,6 @@ def test_subworkflows_remove_subworkflow(self): mod_json_before = mod_json_obj.get_modules_json() assert self.subworkflow_remove.remove("bam_sort_stats_samtools") mod_json_after = mod_json_obj.get_modules_json() - import ipdb - - ipdb.set_trace() assert Path.exists(subworkflow_path) is False assert Path.exists(bam_sort_stats_samtools_path) is False assert Path.exists(bam_stats_samtools_path) is False @@ -32,15 +29,16 @@ def test_subworkflows_remove_one_of_two_subworkflow(self): """Test removing subworkflow and all it's dependencies after installing it""" self.subworkflow_install.install("bam_sort_stats_samtools") self.subworkflow_install.install("bam_stats_samtools") - subworkflow_path = Path(self.subworkflow_install.dir, "subworkflows", "nf-core") bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools") bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") + samtools_stats_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "stats") assert self.subworkflow_remove.remove("bam_sort_stats_samtools") assert Path.exists(subworkflow_path) is True assert Path.exists(bam_sort_stats_samtools_path) is False assert Path.exists(bam_stats_samtools_path) is True - assert Path.exists(samtools_index_path) is True + assert Path.exists(samtools_index_path) is False + assert Path.exists(samtools_stats_path) is True diff --git a/tests/test_modules.py b/tests/test_modules.py index f14d256dc4..be503b75dd 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -160,6 +160,7 @@ def test_modulesrepo_class(self): test_modules_list_remote, test_modules_list_remote_gitlab, ) + from .modules.modules_json import ( test_get_modules_json, test_mod_json_create, @@ -188,6 +189,7 @@ def test_modulesrepo_class(self): test_create_patch_update_fail, test_create_patch_update_success, ) + from .modules.remove import ( test_modules_remove_trimgalore, test_modules_remove_trimgalore_uninstalled, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 59e508111d..4131f0f659 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -91,8 +91,8 @@ def setUp(self): from .subworkflows.remove import ( test_subworkflows_remove_one_of_two_subworkflow, test_subworkflows_remove_subworkflow, - test_subworkflows_remove_uninstalled_subworkflow, ) + from .subworkflows.subworkflows_test import ( test_subworkflows_test_check_inputs, test_subworkflows_test_no_installed_subworkflows, From 52f1fe20505a2d0416ce84be90628978bda9d55e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Nov 2022 11:47:17 +0100 Subject: [PATCH 533/854] fix isort --- tests/subworkflows/remove.py | 1 + tests/test_modules.py | 2 -- tests/test_subworkflows.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/subworkflows/remove.py b/tests/subworkflows/remove.py index 65be9c3d22..a4f525a96d 100644 --- a/tests/subworkflows/remove.py +++ b/tests/subworkflows/remove.py @@ -1,4 +1,5 @@ from pathlib import Path + from nf_core.modules.modules_json import ModulesJson diff --git a/tests/test_modules.py b/tests/test_modules.py index be503b75dd..f14d256dc4 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -160,7 +160,6 @@ def test_modulesrepo_class(self): test_modules_list_remote, test_modules_list_remote_gitlab, ) - from .modules.modules_json import ( test_get_modules_json, test_mod_json_create, @@ -189,7 +188,6 @@ def test_modulesrepo_class(self): test_create_patch_update_fail, test_create_patch_update_success, ) - from .modules.remove import ( test_modules_remove_trimgalore, test_modules_remove_trimgalore_uninstalled, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 4131f0f659..7ec6390c6c 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -92,7 +92,6 @@ def setUp(self): test_subworkflows_remove_one_of_two_subworkflow, test_subworkflows_remove_subworkflow, ) - from .subworkflows.subworkflows_test import ( test_subworkflows_test_check_inputs, test_subworkflows_test_no_installed_subworkflows, From 61e3271e6579eb4448b81b9493225fdabae913bf Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Nov 2022 14:42:14 +0100 Subject: [PATCH 534/854] update rich-click info commands --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ab906bd658..1f28da1433 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -58,7 +58,7 @@ "nf-core subworkflows": [ { "name": "For pipelines", - "commands": ["install"], + "commands": ["install", "list", "remove"], }, { "name": "Developing new subworkflows", From e1ddb3e168592ad613a8fd5a1e1134f506df225c Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 15 Nov 2022 14:09:12 +0000 Subject: [PATCH 535/854] [automated] Fix code linting --- nf_core/subworkflows/list.py | 2 +- tests/subworkflows/list.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/subworkflows/list.py b/nf_core/subworkflows/list.py index f1e80c93a3..ddf144ee00 100644 --- a/nf_core/subworkflows/list.py +++ b/nf_core/subworkflows/list.py @@ -7,4 +7,4 @@ class SubworkflowList(ComponentList): def __init__(self, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False): - super().__init__("subworkflows", pipeline_dir, remote, remote_url, branch, no_pull) \ No newline at end of file + super().__init__("subworkflows", pipeline_dir, remote, remote_url, branch, no_pull) diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index 9b5fb5ee28..8daf5fb599 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -46,4 +46,4 @@ def test_subworkflows_install_gitlab_and_list_subworkflows(self): console = Console(record=True) console.print(listed_subworkflows) output = console.export_text() - assert "bam_stats" in output \ No newline at end of file + assert "bam_stats" in output From e50934796deb88033fbfca254236bef9a9290af4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 15 Nov 2022 15:12:52 +0100 Subject: [PATCH 536/854] Fix hide-progress bug Follow on from PR nf-core/tools#2016 --- nf_core/__main__.py | 11 +++++------ nf_core/modules/lint/__init__.py | 7 ++++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e7087b4931..3ea2e62380 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -738,17 +738,16 @@ def lint(ctx, tool, dir, key, all, fail_warned, local, passed, fix_version): # try: module_lint = nf_core.modules.ModuleLint( dir, - fail_warned, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], - ctx.obj["hide_progress"], + fail_warned=fail_warned, + remote_url=ctx.obj["modules_repo_url"], + branch=ctx.obj["modules_repo_branch"], + no_pull=ctx.obj["modules_repo_no_pull"], + hide_progress=ctx.obj["hide_progress"], ) module_lint.lint( module=tool, key=key, all_modules=all, - hide_progress=ctx.obj["hide_progress"], print_results=True, local=local, show_passed=passed, diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 64251d8e8c..de50770ebb 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -72,7 +72,12 @@ def __init__( hide_progress=False, ): super().__init__( - "modules", dir=dir, remote_url=remote_url, branch=branch, no_pull=no_pull, hide_progress=hide_progress + "modules", + dir=dir, + remote_url=remote_url, + branch=branch, + no_pull=no_pull, + hide_progress=hide_progress, ) self.fail_warned = fail_warned From 1f0d1a06d9b8d2d15a5a8f8074b68acc1c1d2b12 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 15 Nov 2022 16:33:55 +0100 Subject: [PATCH 537/854] fix method name and add missing argument --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cc52ee896c..8786c38fb0 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -318,7 +318,7 @@ def determine_branches_and_shas(self, component_type, install_dir, remote_url, c # Clean up the modules/subworkflows we were unable to find the sha for for component in sb_local: log.debug(f"Moving {component_type[:-1]} '{Path(install_dir, component)}' to 'local' directory") - self.move_module_to_local(component, install_dir) + self.move_component_to_local(component_type, component, install_dir) for component in dead_components: log.debug(f"Removing {component_type[:-1]} {Path(install_dir, component)}'") From 5cc22476e5182c9d0a1f0d13973e40e42bb156eb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 15 Nov 2022 16:47:18 +0100 Subject: [PATCH 538/854] Properly group subcommands in help text --- nf_core/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3ea2e62380..48f33bdb63 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -42,7 +42,7 @@ }, { "name": "Commands for developers", - "commands": ["create", "lint", "modules", "schema", "bump-version", "sync"], + "commands": ["create", "lint", "modules", "subworkflows", "schema", "bump-version", "sync"], }, ], "nf-core modules": [ @@ -58,7 +58,7 @@ "nf-core subworkflows": [ { "name": "For pipelines", - "commands": ["install"], + "commands": ["list", "install"], }, { "name": "Developing new subworkflows", From 85e39f9ea2e40d1f0df8290c00779b5702650161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 15 Nov 2022 16:57:12 +0100 Subject: [PATCH 539/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/remove.py | 2 +- nf_core/modules/modules_json.py | 1 + nf_core/subworkflows/__init__.py | 2 +- nf_core/subworkflows/remove.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index b717ad599e..6b79523b4f 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -48,7 +48,7 @@ def remove(self, component): # Verify that the module/subworkflow is actually installed if not module_dir.exists(): - log.error(f"Module directory does not exist: '{module_dir}'") + log.error(f"{self.component_type[:-1]}.title() directory does not exist: '{module_dir}'") if modules_json.module_present(component, self.modules_repo.remote_url, repo_path): log.error(f"Found entry for '{component}' in 'modules.json'. Removing...") diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 7438526560..39bb5fa801 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -701,6 +701,7 @@ def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=N self.modules_json["repos"][repo_url].pop(component_type) return True self.dump() + # Don't remove component entry from modules.json return False else: log.error( diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index ebb2e3821e..ac4afd23a8 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,6 +1,6 @@ from .create import SubworkflowCreate from .install import SubworkflowInstall from .list import SubworkflowList -from .remove import SubworkflowsRemove +from .remove import SubworkflowRemove from .subworkflows_test import SubworkflowsTest from .test_yml_builder import SubworkflowTestYmlBuilder diff --git a/nf_core/subworkflows/remove.py b/nf_core/subworkflows/remove.py index bd538a4e5a..fcd54095f6 100644 --- a/nf_core/subworkflows/remove.py +++ b/nf_core/subworkflows/remove.py @@ -5,6 +5,6 @@ log = logging.getLogger(__name__) -class SubworkflowsRemove(ComponentRemove): +class SubworkflowRemove(ComponentRemove): def __init__(self, pipeline_dir): super().__init__("subworkflows", pipeline_dir) From 915e6e3fd53456aa2db389d6cf950030b8bcb708 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Nov 2022 17:00:17 +0100 Subject: [PATCH 540/854] apply suggestions from review --- nf_core/__main__.py | 14 ++---------- nf_core/components/components_command.py | 27 ++++++++++++++++++++++++ nf_core/components/remove.py | 25 +++++++++++++--------- nf_core/lint/orphan_include.py | 0 nf_core/modules/modules_json.py | 3 ++- 5 files changed, 46 insertions(+), 23 deletions(-) create mode 100644 nf_core/lint/orphan_include.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 1f28da1433..5e0908761e 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -632,12 +632,7 @@ def remove(ctx, dir, tool): Remove a module from a pipeline. """ try: - module_remove = nf_core.modules.ModuleRemove( - dir, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], - ) + module_remove = nf_core.modules.ModuleRemove(dir) module_remove.remove(tool) except (UserWarning, LookupError) as e: log.critical(e) @@ -1079,12 +1074,7 @@ def remove(ctx, dir, subworkflow): Remove a subworkflow from a pipeline. """ try: - module_remove = nf_core.subworkflows.SubworkflowRemove( - dir, - ctx.obj["modules_repo_url"], - ctx.obj["modules_repo_branch"], - ctx.obj["modules_repo_no_pull"], - ) + module_remove = nf_core.subworkflows.SubworkflowsRemove(dir) module_remove.remove(subworkflow) except (UserWarning, LookupError) as e: log.critical(e) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 9a7511ca49..c832daae6c 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -233,3 +233,30 @@ def check_patch_paths(self, patch_path, module_name): self.modules_repo.repo_path ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) modules_json.dump() + + def check_if_in_include_stmts(self, component_path): + """ + Checks for include statements in the main.nf file of the pipeline and a list of line numbers where the component is included + Args: + component_path (str): The path to the module/subworkflow + + Returns: + (list): A list of dictionaries, with the workflow file and the line number where the component is included + """ + include_stmts = [] + if self.repo_type == "pipeline": + workflow_files = Path(self.dir, "workflows").glob("*.nf") + for workflow_file in workflow_files: + with open(workflow_file, "r") as fh: + for line in fh.readlines(): + if f"'.{Path(component_path,'main')}'" in line: + include_stmts.append( + { + "file": workflow_file, + "line": line, + } + ) + return include_stmts + else: + log.debug("Not a pipeline repository, skipping check for include statements") + return include_stmts diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index b717ad599e..a5daf36332 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -18,6 +18,12 @@ def remove(self, component): """ Remove an already installed module/subworkflow This command only works for modules/subworkflows that are installed from 'nf-core/modules' + + Args: + component (str): Name of the component to remove + + Returns: + bool: True if any item has been removed, False if not """ if self.repo_type == "modules": log.error(f"You cannot remove a {self.component_type[:-1]} in a clone of nf-core/modules") @@ -40,15 +46,15 @@ def remove(self, component): ).unsafe_ask() # Get the module/subworkflow directory - module_dir = Path(self.dir, self.component_type, repo_path, component) + component_dir = Path(self.dir, self.component_type, repo_path, component) # Load the modules.json file modules_json = ModulesJson(self.dir) modules_json.load() # Verify that the module/subworkflow is actually installed - if not module_dir.exists(): - log.error(f"Module directory does not exist: '{module_dir}'") + if not component_dir.exists(): + log.error(f"Module directory does not exist: '{component_dir}'") if modules_json.module_present(component, self.modules_repo.remote_url, repo_path): log.error(f"Found entry for '{component}' in 'modules.json'. Removing...") @@ -59,12 +65,11 @@ def remove(self, component): dependent_components = {component: self.component_type} if self.component_type == "subworkflows": removed_by = component - if removed_by is not None: - dependent_components.update( - modules_json.get_dependent_components( - self.component_type, component, self.modules_repo.remote_url, repo_path, dependent_components - ) + dependent_components.update( + modules_json.get_dependent_components( + self.component_type, component, self.modules_repo.remote_url, repo_path, dependent_components ) + ) # remove all dependent components based on installed_by entry # Remove entry from modules.json removed = False @@ -76,9 +81,9 @@ def remove(self, component): repo_path, removed_by=removed_by, ) + removed_component_dir = Path(self.dir, component_type, repo_path, component_name) # Remove the component files if the entry was removed from modules.json if removed_component: - component_dir = Path(self.dir, component_type, repo_path, component_name) - removed = True if self.clear_component_dir(component, component_dir) or removed else False + removed = True if self.clear_component_dir(component, removed_component_dir) or removed else False return removed diff --git a/nf_core/lint/orphan_include.py b/nf_core/lint/orphan_include.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 7438526560..46a2b6b9a5 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -704,7 +704,8 @@ def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=N return False else: log.error( - f"Could not find 'installed_by' entry for '{removed_by}' in 'modules.json' file. Did you install it first?" + f"{name} was installed by {repo_entry[component_type][install_dir][name]['installed_by']} and can't be removed individually." + f"When you remove {repo_entry[component_type][install_dir][name]['installed_by']}, {name} will be removed as well." ) else: log.warning( From 2624ca73260e22553d9623dfb2d7f8c1893dbf4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 15 Nov 2022 17:37:23 +0100 Subject: [PATCH 541/854] Update tests/test_subworkflows.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/test_subworkflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 7ec6390c6c..cf1df4208c 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -62,7 +62,7 @@ def setUp(self): self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) # Set up remove objects - self.subworkflow_remove = nf_core.subworkflows.SubworkflowsRemove(self.pipeline_dir) + self.subworkflow_remove = nf_core.subworkflows.SubworkflowRemove(self.pipeline_dir) ############################################ # Test of the individual modules commands. # From 98ffb73ca423587842982ec3429f8da3efbd7093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 16 Nov 2022 09:15:03 +0100 Subject: [PATCH 542/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/components/update.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index d59f94610e..47f9cc1258 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -61,7 +61,7 @@ def _parameter_checks(self): raise UserWarning(f"Either a {self.component_type[:-1]} or the '--all' flag can be specified, not both.") if self.repo_type == "modules": - raise UserWarning(f"{self.component_type.title()} in clones of nf-core/modules can not be updated.") + raise UserWarning(f"{self.component_type.title()} can not be updated in clones of the nf-core/modules repository.") if self.prompt and self.sha is not None: raise UserWarning("Cannot use '--sha' and '--prompt' at the same time.") @@ -79,7 +79,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr component (str): The name of the module/subworkflow to update. Returns: - bool: True if the update was successful, False otherwise. + (bool): True if the update was successful, False otherwise. """ self.component = component if updated is None: @@ -222,7 +222,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr ) updated.append(component) except UserWarning as e: - if str(e) != "Module is unchanged": + if str(e) != "{self.component_type[:-1].title()} is unchanged": raise else: updated.append(component) @@ -309,7 +309,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr f" [bold magenta italic]git apply {self.save_diff_fn} [/]" ) elif not all_patches_successful and not silent: - log.info(f"Updates complete. Please apply failed patch{plural_es(components_info)} manually") + log.info(f"Updates complete. Please apply failed patch{plural_es(components_info)} manually.") elif not silent: log.info("Updates complete :sparkles:") From 449885f04de8f809683cbfee2ea09f5fdef748d7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:17:40 +0100 Subject: [PATCH 543/854] add command to rich-click --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3d1c5cba91..3420af0483 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -58,7 +58,7 @@ "nf-core subworkflows": [ { "name": "For pipelines", - "commands": ["install"], + "commands": ["install", "update"], }, { "name": "Developing new subworkflows", From 9383c77fcc18d094321746d28dc5bb78252f3f3b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:23:27 +0100 Subject: [PATCH 544/854] add default to questionary.select --- nf_core/components/update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index d59f94610e..e878162ff8 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -129,6 +129,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr {"name": "Preview diff in terminal, choose whether to update files", "value": 1}, {"name": "Just write diffs to a patch file", "value": 2}, ], + default={"name": "No previews, just update everything", "value": 0}, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() From 33ef4b64af350c144ad5f431b1548dd6393288eb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:31:09 +0100 Subject: [PATCH 545/854] change --recursive by --update-deps --- nf_core/__main__.py | 16 ++++++++-------- nf_core/components/update.py | 8 ++++---- nf_core/modules/update.py | 4 ++-- nf_core/subworkflows/update.py | 4 ++-- tests/modules/patch.py | 9 +++++++-- tests/modules/update.py | 4 ++-- tests/subworkflows/update.py | 8 ++++---- 7 files changed, 29 insertions(+), 24 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3420af0483..59ba06790e 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -555,13 +555,13 @@ def install(ctx, tool, dir, prompt, force, sha): help="Save diffs to a file instead of updating in place", ) @click.option( - "-r", - "--recursive", + "-u", + "--update-deps", is_flag=True, default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, recursive): +def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, update_deps): """ Update DSL2 modules within a pipeline. @@ -576,7 +576,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, recursiv all, preview, save_diff, - recursive, + update - deps, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], @@ -1095,13 +1095,13 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin help="Save diffs to a file instead of updating in place", ) @click.option( - "-r", - "--recursive", + "-u", + "--update-deps", is_flag=True, default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, recursive): +def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, update_deps): """ Update DSL2 subworkflow within a pipeline. @@ -1116,7 +1116,7 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, r all, preview, save_diff, - recursive, + update - deps, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], diff --git a/nf_core/components/update.py b/nf_core/components/update.py index e878162ff8..08277d571f 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -29,7 +29,7 @@ def __init__( update_all=False, show_diff=None, save_diff_fn=None, - recursive=False, + update_deps=False, remote_url=None, branch=None, no_pull=False, @@ -41,7 +41,7 @@ def __init__( self.update_all = update_all self.show_diff = show_diff self.save_diff_fn = save_diff_fn - self.recursive = recursive + self.update_deps = update - deps self.component = None self.update_config = None self.modules_json = ModulesJson(self.dir) @@ -234,7 +234,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr "It is advised to keep all your modules and subworkflows up to date.\n" "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" ) - if self.recursive: + if self.update - deps: recursive_update = True else: recursive_update = questionary.confirm( @@ -278,7 +278,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr "It is advised to keep all your modules and subworkflows up to date.\n" "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" ) - if self.recursive: + if self.update - deps: recursive_update = True else: recursive_update = questionary.confirm( diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 350401c43e..d07b9482ce 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -11,7 +11,7 @@ def __init__( update_all=False, show_diff=None, save_diff_fn=None, - recursive=False, + update_deps=False, remote_url=None, branch=None, no_pull=False, @@ -25,7 +25,7 @@ def __init__( update_all, show_diff, save_diff_fn, - recursive, + update - deps, remote_url, branch, no_pull, diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py index 200ade9c50..17a3cd39f7 100644 --- a/nf_core/subworkflows/update.py +++ b/nf_core/subworkflows/update.py @@ -11,7 +11,7 @@ def __init__( update_all=False, show_diff=None, save_diff_fn=None, - recursive=False, + update_deps=False, remote_url=None, branch=None, no_pull=False, @@ -25,7 +25,7 @@ def __init__( update_all, show_diff, save_diff_fn, - recursive, + update - deps, remote_url, branch, no_pull, diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 100afe8731..8bad4c59a3 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -235,7 +235,12 @@ def test_create_patch_update_success(self): # Update the module update_obj = nf_core.modules.ModuleUpdate( - self.pipeline_dir, sha=SUCCEED_SHA, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=PATCH_BRANCH + self.pipeline_dir, + sha=SUCCEED_SHA, + show_diff=False, + update_deps=True, + remote_url=GITLAB_URL, + branch=PATCH_BRANCH, ) assert update_obj.update(BISMARK_ALIGN) @@ -294,7 +299,7 @@ def test_create_patch_update_fail(self): patch_contents = fh.read() update_obj = nf_core.modules.ModuleUpdate( - self.pipeline_dir, sha=FAIL_SHA, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=PATCH_BRANCH + self.pipeline_dir, sha=FAIL_SHA, show_diff=False, update_deps=True, remote_url=GITLAB_URL, branch=PATCH_BRANCH ) update_obj.update(BISMARK_ALIGN) diff --git a/tests/modules/update.py b/tests/modules/update.py index a4121f917c..466bd61185 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -46,7 +46,7 @@ def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" assert self.mods_install_old.install("trimgalore") update_obj = ModuleUpdate( - self.pipeline_dir, show_diff=False, recursive=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH + self.pipeline_dir, show_diff=False, update_deps=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH ) # Copy the module files and check that they are affected by the update @@ -231,7 +231,7 @@ def test_update_different_branch_single_module(self): assert install_obj.install("fastp") update_obj = ModuleUpdate( - self.pipeline_dir, recursive=True, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False + self.pipeline_dir, update_deps=True, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, show_diff=False ) update_obj.update("fastp") diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 0b061e0acf..8626385072 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -32,7 +32,7 @@ def test_install_and_update(self): def test_install_at_hash_and_update(self): """Installs an old version of a subworkflow in the pipeline and updates it""" assert self.sw_install_old.install("fastq_align_bowtie2") - update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False, recursive=True) + update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False, update_deps=True) old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Copy the sw files and check that they are affected by the update @@ -61,7 +61,7 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a sw in the pipeline and updates it. Save differences to a file.""" assert self.sw_install_old.install("fastq_align_bowtie2") patch_path = Path(self.pipeline_dir, "fastq_align_bowtie2.patch") - update_obj = SubworkflowUpdate(self.pipeline_dir, save_diff_fn=patch_path, recursive=True) + update_obj = SubworkflowUpdate(self.pipeline_dir, save_diff_fn=patch_path, update_deps=True) # Copy the sw files and check that they are affected by the update tmpdir = tempfile.mkdtemp() @@ -231,7 +231,7 @@ def test_update_all_linked_components_from_subworkflow(self): shutil.copytree(modules_path, Path(tmpdir, "modules")) # Update fastq_align_bowtie2 and all modules and subworkflows used by that - update_obj = SubworkflowUpdate(self.pipeline_dir, recursive=True, show_diff=False) + update_obj = SubworkflowUpdate(self.pipeline_dir, update_deps=True, show_diff=False) assert update_obj.update("fastq_align_bowtie2") is True mod_json = ModulesJson(self.pipeline_dir).get_modules_json() @@ -276,7 +276,7 @@ def test_update_all_subworkflows_from_module(self): shutil.copytree(sw_path, Path(tmpdir, "fastq_align_bowtie2")) # Update bowtie2/align and all subworkflows using it - update_obj = ModuleUpdate(self.pipeline_dir, recursive=True, show_diff=False) + update_obj = ModuleUpdate(self.pipeline_dir, update_deps=True, show_diff=False) assert update_obj.update("bowtie2/align") is True mod_json = ModulesJson(self.pipeline_dir).get_modules_json() From 637a036c27355e0cfff0b3e181b9713ece27cc04 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:33:49 +0100 Subject: [PATCH 546/854] fix lint mess --- nf_core/components/update.py | 6 +++--- nf_core/modules/update.py | 2 +- nf_core/subworkflows/update.py | 2 +- tests/test_subworkflows.py | 7 +------ 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 08277d571f..99c8f85349 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -41,7 +41,7 @@ def __init__( self.update_all = update_all self.show_diff = show_diff self.save_diff_fn = save_diff_fn - self.update_deps = update - deps + self.update_deps = update_deps self.component = None self.update_config = None self.modules_json = ModulesJson(self.dir) @@ -234,7 +234,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr "It is advised to keep all your modules and subworkflows up to date.\n" "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" ) - if self.update - deps: + if self.update_deps: recursive_update = True else: recursive_update = questionary.confirm( @@ -278,7 +278,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr "It is advised to keep all your modules and subworkflows up to date.\n" "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" ) - if self.update - deps: + if self.update_deps: recursive_update = True else: recursive_update = questionary.confirm( diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index d07b9482ce..9d53bf2017 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -25,7 +25,7 @@ def __init__( update_all, show_diff, save_diff_fn, - update - deps, + update_deps, remote_url, branch, no_pull, diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py index 17a3cd39f7..3cd4ad59fd 100644 --- a/nf_core/subworkflows/update.py +++ b/nf_core/subworkflows/update.py @@ -25,7 +25,7 @@ def __init__( update_all, show_diff, save_diff_fn, - update - deps, + update_deps, remote_url, branch, no_pull, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index abdd550ecb..d3eaa14a86 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -11,12 +11,7 @@ import nf_core.modules import nf_core.subworkflows -from .utils import ( - GITLAB_SUBWORKFLOWS_BRANCH, - GITLAB_URL, - OLD_SUBWORKFLOWS_SHA, - mock_api_calls, -) +from .utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, OLD_SUBWORKFLOWS_SHA def create_modules_repo_dummy(tmp_dir): From 0cd49960836a11d9416df9848c907586cc5b64ca Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:38:30 +0100 Subject: [PATCH 547/854] only warn about other modules/subworkflows to update if there are some --- nf_core/components/update.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 99c8f85349..f3a11e8c30 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -228,7 +228,10 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr else: updated.append(component) recursive_update = True - if not silent: + modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update( + component, modules_repo + ) + if not silent and len(modules_to_update + subworkflows_to_update) > 0: log.warning( f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be added to the same diff file.\n" "It is advised to keep all your modules and subworkflows up to date.\n" @@ -242,9 +245,11 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr default=True, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - if recursive_update: + if recursive_update and len(modules_to_update + subworkflows_to_update) > 0: # Write all the differences of linked componenets to a diff file - self.update_linked_components(component, modules_repo, updated, check_diff_exist=False) + self.update_linked_components( + modules_to_update, subworkflows_to_update, updated, check_diff_exist=False + ) elif self.show_diff: ModulesDiffer.print_diff( @@ -272,7 +277,10 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type) updated.append(component) recursive_update = True - if not silent and not self.update_all: + modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update( + component, modules_repo + ) + if not silent and not self.update_all and len(modules_to_update + subworkflows_to_update) > 0: log.warning( f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n" "It is advised to keep all your modules and subworkflows up to date.\n" @@ -286,9 +294,9 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr default=True, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - if recursive_update: + if recursive_update and len(modules_to_update + subworkflows_to_update) > 0: # Update linked components - self.update_linked_components(component, modules_repo, updated) + self.update_linked_components(modules_to_update, subworkflows_to_update, updated) else: # Don't save to a file, just iteratively update the variable self.modules_json.update( @@ -818,11 +826,10 @@ def get_modules_subworkflows_to_update(self, component, modules_repo): return modules_to_update, subworkflows_to_update - def update_linked_components(self, component, modules_repo, updated=None, check_diff_exist=True): + def update_linked_components(self, modules_to_update, subworkflows_to_update, updated=None, check_diff_exist=True): """ Update modules and subworkflows linked to the component being updated. """ - modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update(component, modules_repo) for s_update in subworkflows_to_update: if s_update in updated: continue From 6b311cd0363eaf37380fa1000a174304f0d30ef2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:43:57 +0100 Subject: [PATCH 548/854] change sw_install by subworkflow_install --- tests/subworkflows/install.py | 32 ++++++++++++++++---------------- tests/subworkflows/list.py | 4 ++-- tests/subworkflows/update.py | 20 ++++++++++---------- tests/test_subworkflows.py | 6 +++--- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index f394ef4066..94b2d79ba0 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -16,35 +16,35 @@ def test_subworkflow_install_nopipeline(self): """Test installing a subworkflow - no pipeline given""" - self.sw_install.dir = None - assert self.sw_install.install("foo") is False + self.subworkflow_install.dir = None + assert self.subworkflow_install.install("foo") is False @with_temporary_folder def test_subworkflows_install_emptypipeline(self, tmpdir): """Test installing a subworkflow - empty dir given""" os.mkdir(os.path.join(tmpdir, "nf-core-pipe")) - self.sw_install.dir = os.path.join(tmpdir, "nf-core-pipe") + self.subworkflow_install.dir = os.path.join(tmpdir, "nf-core-pipe") with pytest.raises(UserWarning) as excinfo: - self.sw_install.install("foo") + self.subworkflow_install.install("foo") assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) def test_subworkflows_install_nosubworkflow(self): """Test installing a subworkflow - unrecognised subworkflow given""" - assert self.sw_install.install("foo") is False + assert self.subworkflow_install.install("foo") is False def test_subworkflows_install_bam_sort_stats_samtools(self): """Test installing a subworkflow - bam_sort_stats_samtools""" - assert self.sw_install.install("bam_sort_stats_samtools") is not False - subworkflow_path = os.path.join(self.sw_install.dir, "subworkflows", "nf-core", "bam_sort_stats_samtools") - sub_subworkflow_path = os.path.join(self.sw_install.dir, "subworkflows", "nf-core", "bam_stats_samtools") - samtools_index_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "index") - samtools_sort_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "sort") - samtools_stats_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "stats") - samtools_idxstats_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "idxstats") - samtools_flagstat_path = os.path.join(self.sw_install.dir, "modules", "nf-core", "samtools", "flagstat") + assert self.subworkflow_install.install("bam_sort_stats_samtools") is not False + subworkflow_path = os.path.join(self.subworkflow_install.dir, "subworkflows", "nf-core", "bam_sort_stats_samtools") + sub_subworkflow_path = os.path.join(self.subworkflow_install.dir, "subworkflows", "nf-core", "bam_stats_samtools") + samtools_index_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") + samtools_sort_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "sort") + samtools_stats_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "stats") + samtools_idxstats_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "idxstats") + samtools_flagstat_path = os.path.join(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "flagstat") assert os.path.exists(subworkflow_path) assert os.path.exists(sub_subworkflow_path) assert os.path.exists(samtools_index_path) @@ -56,13 +56,13 @@ def test_subworkflows_install_bam_sort_stats_samtools(self): def test_subworkflows_install_bam_sort_stats_samtools_twice(self): """Test installing a subworkflow - bam_sort_stats_samtools already there""" - self.sw_install.install("bam_sort_stats_samtools") - assert self.sw_install.install("bam_sort_stats_samtools") is False + self.subworkflow_install.install("bam_sort_stats_samtools") + assert self.subworkflow_install.install("bam_sort_stats_samtools") is False def test_subworkflows_install_from_gitlab(self): """Test installing a subworkflow from GitLab""" - assert self.sw_install_gitlab.install("bam_stats_samtools") is True + assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) assert ( diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index 8daf5fb599..c65999d42c 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -29,7 +29,7 @@ def test_subworkflows_list_remote_gitlab(self): def test_subworkflows_install_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - self.sw_install.install("bam_sort_stats_samtools") + self.subworkflow_install.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() console = Console(record=True) @@ -40,7 +40,7 @@ def test_subworkflows_install_and_list_subworkflows(self): def test_subworkflows_install_gitlab_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - self.sw_install_gitlab.install("bam_sort_stats_samtools") + self.subworkflow_install_gitlab.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() console = Console(record=True) diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 8626385072..549f9366d8 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -16,7 +16,7 @@ def test_install_and_update(self): """Installs a subworkflow in the pipeline and updates it (no change)""" - self.sw_install.install("bam_stats_samtools") + self.subworkflow_install.install("bam_stats_samtools") update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False) # Copy the sw files and check that they are unaffected by the update @@ -31,7 +31,7 @@ def test_install_and_update(self): def test_install_at_hash_and_update(self): """Installs an old version of a subworkflow in the pipeline and updates it""" - assert self.sw_install_old.install("fastq_align_bowtie2") + assert self.subworkflow_install_old.install("fastq_align_bowtie2") update_obj = SubworkflowUpdate(self.pipeline_dir, show_diff=False, update_deps=True) old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() @@ -59,7 +59,7 @@ def test_install_at_hash_and_update(self): def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a sw in the pipeline and updates it. Save differences to a file.""" - assert self.sw_install_old.install("fastq_align_bowtie2") + assert self.subworkflow_install_old.install("fastq_align_bowtie2") patch_path = Path(self.pipeline_dir, "fastq_align_bowtie2.patch") update_obj = SubworkflowUpdate(self.pipeline_dir, save_diff_fn=patch_path, update_deps=True) @@ -82,7 +82,7 @@ def test_install_at_hash_and_update_and_save_diff_to_file(self): def test_update_all(self): """Updates all subworkflows present in the pipeline""" # Install subworkflows fastq_align_bowtie2, bam_sort_stats_samtools, bam_stats_samtools - self.sw_install.install("fastq_align_bowtie2") + self.subworkflow_install.install("fastq_align_bowtie2") # Update all subworkflows update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=True, show_diff=False) assert update_obj.update() is True @@ -100,7 +100,7 @@ def test_update_all(self): def test_update_with_config_fixed_version(self): """Try updating when there are entries in the .nf-core.yml""" # Install subworkflow at the latest version - assert self.sw_install.install("fastq_align_bowtie2") + assert self.subworkflow_install.install("fastq_align_bowtie2") # Fix the subworkflow version in the .nf-core.yml to an old version update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"fastq_align_bowtie2": OLD_SUBWORKFLOWS_SHA}}} @@ -131,7 +131,7 @@ def test_update_with_config_fixed_version(self): def test_update_with_config_dont_update(self): """Try updating when sw is to be ignored""" # Install an old version of fastq_align_bowtie2 - self.sw_install_old.install("fastq_align_bowtie2") + self.subworkflow_install_old.install("fastq_align_bowtie2") # Set the fastq_align_bowtie2 field to no update in the .nf-core.yml update_config = {NF_CORE_MODULES_REMOTE: {NF_CORE_MODULES_NAME: {"fastq_align_bowtie2": False}}} @@ -162,7 +162,7 @@ def test_update_with_config_dont_update(self): def test_update_with_config_fix_all(self): """Fix the version of all nf-core subworkflows""" # Install subworkflow at the latest version - assert self.sw_install.install("fastq_align_bowtie2") + assert self.subworkflow_install.install("fastq_align_bowtie2") # Fix the version of all nf-core subworkflows in the .nf-core.yml to an old version update_config = {NF_CORE_MODULES_REMOTE: OLD_SUBWORKFLOWS_SHA} @@ -192,7 +192,7 @@ def test_update_with_config_fix_all(self): def test_update_with_config_no_updates(self): """Don't update any nf-core subworkflows""" # Install an old version of fastq_align_bowtie2 - self.sw_install_old.install("fastq_align_bowtie2") + self.subworkflow_install_old.install("fastq_align_bowtie2") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Set all repository updates to False @@ -219,7 +219,7 @@ def test_update_with_config_no_updates(self): def test_update_all_linked_components_from_subworkflow(self): """Update a subworkflow and all modules and subworkflows used on it""" # Install an old version of fastq_align_bowtie2 - self.sw_install_old.install("fastq_align_bowtie2") + self.subworkflow_install_old.install("fastq_align_bowtie2") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Copy the sw files and check that they are affected by the update @@ -266,7 +266,7 @@ def test_update_all_linked_components_from_subworkflow(self): def test_update_all_subworkflows_from_module(self): """Update a module and all subworkflows that use this module""" # Install an old version of fastq_align_bowtie2 and thus all modules used by it (bowtie2/align) - self.sw_install_old.install("fastq_align_bowtie2") + self.subworkflow_install_old.install("fastq_align_bowtie2") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Copy the sw files and check that they are affected by the update diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index d3eaa14a86..56b4549cd0 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -56,11 +56,11 @@ def setUp(self): self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) # Set up install objects - self.sw_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=False) - self.sw_install_gitlab = nf_core.subworkflows.SubworkflowInstall( + self.subworkflow_install = nf_core.subworkflows.SubworkflowInstall(self.pipeline_dir, prompt=False, force=False) + self.subworkflow_install_gitlab = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH ) - self.sw_install_old = nf_core.subworkflows.SubworkflowInstall( + self.subworkflow_install_old = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, force=False, From 166ad3e57ab00f7e57271ddaad380a89ec41c334 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:47:41 +0100 Subject: [PATCH 549/854] modify wrong variable names --- nf_core/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 59ba06790e..7e377ae965 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -576,7 +576,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, update_d all, preview, save_diff, - update - deps, + update_deps, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], @@ -1116,7 +1116,7 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, u all, preview, save_diff, - update - deps, + update_deps, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], From cf31aabf16140a56ff39f058af293c2b62b57e27 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 09:52:21 +0100 Subject: [PATCH 550/854] improve error message --- nf_core/components/update.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index f3a11e8c30..575f8c6885 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -356,7 +356,10 @@ def get_single_component_info(self, component): ).unsafe_ask() # Get component installation directory - install_dir = [dir for dir, m in components if component == m][0] + try: + install_dir = [dir for dir, m in components if component == m][0] + except IndexError: + raise UserWarning(f"{self.component_type[:-1].title()} '{component}' not found in 'modules.json'.") # Check if component is installed before trying to update if component not in choices: From 5c518221abe45e2dc4d2e7f5172a08f617622c14 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 16 Nov 2022 09:03:07 +0000 Subject: [PATCH 551/854] [automated] Fix code linting --- nf_core/components/update.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index e40e6c543c..3336b2f324 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -61,7 +61,9 @@ def _parameter_checks(self): raise UserWarning(f"Either a {self.component_type[:-1]} or the '--all' flag can be specified, not both.") if self.repo_type == "modules": - raise UserWarning(f"{self.component_type.title()} can not be updated in clones of the nf-core/modules repository.") + raise UserWarning( + f"{self.component_type.title()} can not be updated in clones of the nf-core/modules repository." + ) if self.prompt and self.sha is not None: raise UserWarning("Cannot use '--sha' and '--prompt' at the same time.") From a59685700ee897a7bbac2843e2e974f7ea132bf2 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 16 Sep 2022 11:50:51 +0100 Subject: [PATCH 552/854] Extended the hook_url functionality to Slack --- .prettierignore | 1 + CHANGELOG.md | 1 + nf_core/pipeline-template/.prettierignore | 1 + .../pipeline-template/assets/slackreport.json | 34 +++++++++++++++++++ .../lib/NfcoreTemplate.groovy | 10 +++--- .../pipeline-template/nextflow_schema.json | 2 +- .../pipeline-template/workflows/pipeline.nf | 2 +- 7 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 nf_core/pipeline-template/assets/slackreport.json diff --git a/.prettierignore b/.prettierignore index 5d06ce6ae8..bd1a8bee9c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,6 @@ email_template.html adaptivecard.json +slackreport.json docs/api/_build testing diff --git a/CHANGELOG.md b/CHANGELOG.md index a63de2682e..b6b7c8b366 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999)) - Substitute ModulesCommand and SubworkflowsCommand by ComponentsCommand ([#2000](https://github.com/nf-core/tools/pull/2000)) - Don't print source file + line number on logging messages (except when verbose) ([#2015](https://github.com/nf-core/tools/pull/2015)) +- Extended the chat notifications to Slack ([#1829](https://github.com/nf-core/tools/pull/1829)) ### Modules diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore index eb74a5741c..0a949c30b2 100644 --- a/nf_core/pipeline-template/.prettierignore +++ b/nf_core/pipeline-template/.prettierignore @@ -1,5 +1,6 @@ email_template.html adaptivecard.json +slackreport.json .nextflow* work/ data/ diff --git a/nf_core/pipeline-template/assets/slackreport.json b/nf_core/pipeline-template/assets/slackreport.json new file mode 100644 index 0000000000..043d02f275 --- /dev/null +++ b/nf_core/pipeline-template/assets/slackreport.json @@ -0,0 +1,34 @@ +{ + "attachments": [ + { + "fallback": "Plain-text summary of the attachment.", + "color": "<% if (success) { %>good<% } else { %>danger<%} %>", + "author_name": "sanger-tol/readmapping v${version} - ${runName}", + "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", + "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", + "fields": [ + { + "title": "Command used to launch the workflow", + "value": "```${commandLine}```", + "short": false + } + <% + if (!success) { %> + , + { + "title": "Full error message", + "value": "```${errorReport}```", + "short": false + }, + { + "title": "Pipeline configuration", + "value": "<% out << summary.collect{ k,v -> k == "hook_url" ? "_${k}_: (_hidden_)" : ( ( v.class.toString().contains('Path') || ( v.class.toString().contains('String') && v.contains('/') ) ) ? "_${k}_: `${v}`" : (v.class.toString().contains('DateTime') ? ("_${k}_: " + v.format(java.time.format.DateTimeFormatter.ofLocalizedDateTime(java.time.format.FormatStyle.MEDIUM))) : "_${k}_: ${v}") ) }.join(",\n") %>", + "short": false + } + <% } + %> + ], + "footer": "Completed at <% out << dateComplete.format(java.time.format.DateTimeFormatter.ofLocalizedDateTime(java.time.format.FormatStyle.MEDIUM)) %> (duration: ${duration})" + } + ] +} diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 8cd2cc1e9e..ea5f16407e 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -146,10 +146,9 @@ class NfcoreTemplate { } // - // Construct and send adaptive card - // https://adaptivecards.io + // Construct and send a web report as JSON // - public static void adaptivecard(workflow, params, summary_params, projectDir, log) { + public static void webreport(workflow, params, summary_params, projectDir, log) { def hook_url = params.hook_url def summary = [:] @@ -184,7 +183,10 @@ class NfcoreTemplate { // Render the JSON template def engine = new groovy.text.GStringTemplateEngine() - def hf = new File("$projectDir/assets/adaptivecard.json") + // Different JSON depending on the service provider + // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format + def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" + def hf = new File("$projectDir/assets/${json_path}") def json_template = engine.createTemplate(hf).make(msg_fields) def json_message = json_template.toString() diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index d6c455d707..976f2719c8 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -217,7 +217,7 @@ "type": "string", "description": "Incoming hook URL for messaging service", "fa_icon": "fas fa-people-group", - "help_text": "Incoming hook URL for messaging service. Currently, only MS Teams is supported.", + "help_text": "Incoming hook URL for messaging service. Currently, MS Teams and Slack are supported.", "hidden": true }, "multiqc_config": { diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index b39148308b..efbd1f8c57 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -121,7 +121,7 @@ workflow.onComplete { } NfcoreTemplate.summary(workflow, params, log) if (params.hook_url) { - NfcoreTemplate.adaptivecard(workflow, params, summary_params, projectDir, log) + NfcoreTemplate.webreport(workflow, params, summary_params, projectDir, log) } } From de322e60992850876c54955b3e5285e9039532f7 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 16 Sep 2022 14:23:51 +0100 Subject: [PATCH 553/854] Renamed the function to IM_notification to avoid confusion with weblog --- nf_core/pipeline-template/lib/NfcoreTemplate.groovy | 5 +++-- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index ea5f16407e..da3a9dd2cb 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -146,9 +146,10 @@ class NfcoreTemplate { } // - // Construct and send a web report as JSON + // Construct and send a notification to a web server as JSON + // e.g. Microsoft Teams and Slack // - public static void webreport(workflow, params, summary_params, projectDir, log) { + public static void IM_notification(workflow, params, summary_params, projectDir, log) { def hook_url = params.hook_url def summary = [:] diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index efbd1f8c57..2b54789101 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -121,7 +121,7 @@ workflow.onComplete { } NfcoreTemplate.summary(workflow, params, log) if (params.hook_url) { - NfcoreTemplate.webreport(workflow, params, summary_params, projectDir, log) + NfcoreTemplate.IM_notification(workflow, params, summary_params, projectDir, log) } } From 33a09af003f009b8efbffafe90021131ebab33d8 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 16 Nov 2022 09:42:58 +0000 Subject: [PATCH 554/854] Hide the hook URL from the command line too --- nf_core/pipeline-template/lib/NfcoreTemplate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index da3a9dd2cb..57b72d4e6d 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -178,7 +178,7 @@ class NfcoreTemplate { msg_fields['exitStatus'] = workflow.exitStatus msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine + msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") msg_fields['projectDir'] = workflow.projectDir msg_fields['summary'] = summary << misc_fields From 2e07990a8e5e028d56efd1f0d41b7fbdc070fbe3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 16 Nov 2022 10:48:54 +0100 Subject: [PATCH 555/854] revert suggestion that makes tests fail --- nf_core/components/update.py | 6 ++++-- tests/test_subworkflows.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index e40e6c543c..4e2025a158 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -61,7 +61,9 @@ def _parameter_checks(self): raise UserWarning(f"Either a {self.component_type[:-1]} or the '--all' flag can be specified, not both.") if self.repo_type == "modules": - raise UserWarning(f"{self.component_type.title()} can not be updated in clones of the nf-core/modules repository.") + raise UserWarning( + f"{self.component_type.title()} can not be updated in clones of the nf-core/modules repository." + ) if self.prompt and self.sha is not None: raise UserWarning("Cannot use '--sha' and '--prompt' at the same time.") @@ -223,7 +225,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr ) updated.append(component) except UserWarning as e: - if str(e) != "{self.component_type[:-1].title()} is unchanged": + if str(e) != "Module is unchanged": raise else: updated.append(component) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 56b4549cd0..bd6ab0f9fa 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -67,9 +67,9 @@ def setUp(self): sha=OLD_SUBWORKFLOWS_SHA, ) - ############################################ - # Test of the individual modules commands. # - ############################################ + ################################################ + # Test of the individual subworkflow commands. # + ################################################ from .subworkflows.create import ( test_subworkflows_create_fail_exists, From 9a1e311b9424719de0bae8a1e2318874a2d6d950 Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Wed, 16 Nov 2022 20:21:35 +0100 Subject: [PATCH 556/854] fix subworkflows info command --- nf_core/__main__.py | 2 +- nf_core/subworkflows/info.py | 22 ++++++---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 1496b5c6c1..ec73bee109 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1024,7 +1024,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin # nf-core subworkflows info @subworkflows.command() @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("tool", type=str, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index 1489364971..e9349cc2f6 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -1,6 +1,5 @@ import logging import os -import shutil from pathlib import Path import questionary @@ -12,6 +11,8 @@ from rich.table import Table from rich.text import Text +import nf_core.utils +from nf_core.components.components_command import ComponentCommand import nf_core.modules.modules_utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE, ModulesRepo @@ -20,7 +21,7 @@ log = logging.getLogger(__name__) -class SubworkflowInfo(object): +class SubworkflowInfo(ComponentCommand): """ Class to print information of a subworkflow. @@ -56,17 +57,7 @@ class SubworkflowInfo(object): """ def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): - # super().__init__(pipeline_dir, remote_url, branch, no_pull) - self.dir = pipeline_dir - self.modules_repo = ModulesRepo(remote_url, branch, no_pull) - try: - if self.dir: - self.dir, self.repo_type = get_repo_type(self.dir) - else: - self.repo_type = None - except LookupError as e: - raise UserWarning(e) - + super().__init__("subworkflows", pipeline_dir, remote_url, branch, no_pull) self.meta = None self.local_path = None self.remote_location = None @@ -111,7 +102,7 @@ def init_mod_name(self, subworkflow): if subworkflows is None: raise UserWarning(f"No subworkflow installed from '{self.modules_repo.remote_url}'") else: - subworkflows = self.modules_repo.get_avail_subworkflows() + subworkflows = self.modules_repo.get_avail_components(self.component_type) subworkflow = questionary.autocomplete( "Please select a subworkflow", choices=subworkflows, style=nf_core.utils.nfcore_question_style ).unsafe_ask() @@ -149,7 +140,6 @@ def get_local_yaml(self): if self.repo_type == "pipeline": # Try to find and load the meta.yml file - repo_name = self.modules_repo.repo_path module_base_path = os.path.join(self.dir, "subworkflows") # Check that we have any subworkflows installed from this repo subworkflows = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) @@ -189,7 +179,7 @@ def get_remote_yaml(self): dict or bool: Parsed meta.yml found, False otherwise """ # Check if our requested module is there - if self.subworkflow not in self.modules_repo.get_avail_subworkflows(): + if self.subworkflow not in self.modules_repo.get_avail_components(self.component_type): return False file_contents = self.modules_repo.get_subworkflow_meta_yml(self.subworkflow) From dc60869cfa4627fcbd7d85f4e48ef0810ce29cf1 Mon Sep 17 00:00:00 2001 From: bjlang <> Date: Wed, 16 Nov 2022 20:27:38 +0100 Subject: [PATCH 557/854] Fix isort complaints --- nf_core/subworkflows/info.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index e9349cc2f6..a428ffd311 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -13,7 +13,6 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand -import nf_core.modules.modules_utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE, ModulesRepo from nf_core.modules.modules_utils import get_repo_type From e39bb3603ae0278e064ebe502552afd1a6fdd50e Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 09:29:01 +0100 Subject: [PATCH 558/854] check for orphan include statements --- nf_core/components/components_command.py | 23 +++++++------ nf_core/components/remove.py | 41 +++++++++++++++++++++--- nf_core/modules/modules_json.py | 11 +++---- tests/subworkflows/remove.py | 38 ++++++++++++++++++++-- tests/test_subworkflows.py | 2 ++ 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index c832daae6c..ebf8aa4586 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -1,4 +1,5 @@ import logging +import mmap import os import shutil from pathlib import Path @@ -243,19 +244,23 @@ def check_if_in_include_stmts(self, component_path): Returns: (list): A list of dictionaries, with the workflow file and the line number where the component is included """ - include_stmts = [] + include_stmts = {} if self.repo_type == "pipeline": workflow_files = Path(self.dir, "workflows").glob("*.nf") for workflow_file in workflow_files: with open(workflow_file, "r") as fh: - for line in fh.readlines(): - if f"'.{Path(component_path,'main')}'" in line: - include_stmts.append( - { - "file": workflow_file, - "line": line, - } - ) + # Check if component path is in the file using mmap + with mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) as s: + if s.find(component_path.encode()) != -1: + # If the component path is in the file, check for include statements + for i, line in enumerate(fh): + if line.startswith("include") and component_path in line: + if str(workflow_file) not in include_stmts: + include_stmts[str(workflow_file)] = [] + include_stmts[str(workflow_file)].append( + {"line_number": i + 1, "line": line.rstrip()} + ) + return include_stmts else: log.debug("Not a pipeline repository, skipping check for include statements") diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index a5daf36332..296ad0618c 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -2,6 +2,9 @@ from pathlib import Path import questionary +from rich.console import Console +from rich.rule import Rule +from rich.syntax import Syntax import nf_core.utils from nf_core.components.components_command import ComponentCommand @@ -73,6 +76,7 @@ def remove(self, component): # remove all dependent components based on installed_by entry # Remove entry from modules.json removed = False + removed_components = [] for component_name, component_type in dependent_components.items(): removed_component = modules_json.remove_entry( component_type, @@ -81,9 +85,38 @@ def remove(self, component): repo_path, removed_by=removed_by, ) - removed_component_dir = Path(self.dir, component_type, repo_path, component_name) - # Remove the component files if the entry was removed from modules.json + removed_component_dir = Path(component_type, repo_path, component_name) if removed_component: - removed = True if self.clear_component_dir(component, removed_component_dir) or removed else False - + if self.component_type == "subworkflows" and component_name != component: + # check if one of the dependent module/subworkflow has been manually included in the pipeline + include_stmts = self.check_if_in_include_stmts(str(removed_component_dir)) + if include_stmts: + # print the include statements + log.warn( + f"The {component_type[:-1]} '{component_name}' is still included in the following workflow file{nf_core.utils.plural_s(include_stmts)}:" + ) + console = Console() + for file, stmts in include_stmts.items(): + console.print(Rule(f"{file}", style="white")) + for stmt in stmts: + console.print( + Syntax( + stmt["line"], + "groovy", + theme="ansi_dark", + line_numbers=True, + start_line=stmt["line_number"], + padding=(0, 0, 1, 1), + ) + ) + + # Remove the component files of all entries removed from modules.json + removed_components.append(component_name.replace("/", "_")) + removed = ( + True + if self.clear_component_dir(component, Path(self.dir, removed_component_dir)) or removed + else False + ) + if removed_components: + log.info(f"Removed files for {', '.join(removed_components)}") return removed diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 46a2b6b9a5..16f15cb0e6 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -93,6 +93,9 @@ def get_component_names_from_repo(self, repos, directory): Args: repos (list): list of repository urls directory (str): modules directory or subworkflows directory + + Returns: + [(str),[(str),(str)]]: list of tuples with repository url, component names and install directory """ names = [ ( @@ -695,18 +698,12 @@ def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=N repo_entry[component_type][install_dir][name]["installed_by"].remove(removed_by) if len(repo_entry[component_type][install_dir][name]["installed_by"]) == 0: self.modules_json["repos"][repo_url][component_type][install_dir].pop(name) - self.dump() if len(repo_entry[component_type][install_dir]) == 0: - self.modules_json["repos"][repo_url].pop(component_type) + self.dump() return True self.dump() return False - else: - log.error( - f"{name} was installed by {repo_entry[component_type][install_dir][name]['installed_by']} and can't be removed individually." - f"When you remove {repo_entry[component_type][install_dir][name]['installed_by']}, {name} will be removed as well." - ) else: log.warning( f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file." diff --git a/tests/subworkflows/remove.py b/tests/subworkflows/remove.py index a4f525a96d..6f4990af96 100644 --- a/tests/subworkflows/remove.py +++ b/tests/subworkflows/remove.py @@ -1,5 +1,7 @@ from pathlib import Path +from rich.console import Console + from nf_core.modules.modules_json import ModulesJson @@ -17,13 +19,44 @@ def test_subworkflows_remove_subworkflow(self): bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") mod_json_obj = ModulesJson(self.pipeline_dir) - mod_json_before = mod_json_obj.get_modules_json() + mod_json_before = ModulesJson(self.pipeline_dir).get_modules_json() assert self.subworkflow_remove.remove("bam_sort_stats_samtools") - mod_json_after = mod_json_obj.get_modules_json() + mod_json_after = ModulesJson(self.pipeline_dir).get_modules_json() assert Path.exists(subworkflow_path) is False assert Path.exists(bam_sort_stats_samtools_path) is False assert Path.exists(bam_stats_samtools_path) is False assert Path.exists(samtools_index_path) is False + assert mod_json_before != mod_json_after + # assert subworkflows key is removed from modules.json + assert "subworkflows" not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"].keys() + assert "samtools/index" not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["modules"].keys() + + +def test_subworkflows_remove_subworkflow_keep_installed_module(self): + """Test removing subworkflow and all it's dependencies after installing it, except for a separately installed module""" + self.subworkflow_install.install("bam_sort_stats_samtools") + self.mods_install.install("samtools/index") + + subworkflow_path = Path(self.subworkflow_install.dir, "subworkflows", "nf-core") + bam_sort_stats_samtools_path = Path(subworkflow_path, "bam_sort_stats_samtools") + bam_stats_samtools_path = Path(subworkflow_path, "bam_stats_samtools") + samtools_index_path = Path(self.subworkflow_install.dir, "modules", "nf-core", "samtools", "index") + + mod_json_before = ModulesJson(self.pipeline_dir).get_modules_json() + assert self.subworkflow_remove.remove("bam_sort_stats_samtools") + mod_json_after = ModulesJson(self.pipeline_dir).get_modules_json() + + assert Path.exists(subworkflow_path) is False + assert Path.exists(bam_sort_stats_samtools_path) is False + assert Path.exists(bam_stats_samtools_path) is False + assert Path.exists(samtools_index_path) is True + assert mod_json_before != mod_json_after + # assert subworkflows key is removed from modules.json + assert "subworkflows" not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"].keys() + assert ( + "samtools/index" + in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["modules"]["nf-core"].keys() + ) def test_subworkflows_remove_one_of_two_subworkflow(self): @@ -43,3 +76,4 @@ def test_subworkflows_remove_one_of_two_subworkflow(self): assert Path.exists(bam_stats_samtools_path) is True assert Path.exists(samtools_index_path) is False assert Path.exists(samtools_stats_path) is True + self.subworkflow_remove.remove("bam_stats_samtools") diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 7ec6390c6c..16c9a194c2 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -57,6 +57,7 @@ def setUp(self): self.subworkflow_install_gitlab = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH ) + self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) @@ -91,6 +92,7 @@ def setUp(self): from .subworkflows.remove import ( test_subworkflows_remove_one_of_two_subworkflow, test_subworkflows_remove_subworkflow, + test_subworkflows_remove_subworkflow_keep_installed_module, ) from .subworkflows.subworkflows_test import ( test_subworkflows_test_check_inputs, From 960e5cfa287c697a7373e4728146c32d26619120 Mon Sep 17 00:00:00 2001 From: Sam Nicholls Date: Thu, 17 Nov 2022 11:12:21 +0000 Subject: [PATCH 559/854] Add --version argument to WorkflowMain --- CHANGELOG.md | 1 + nf_core/lint/nextflow_config.py | 1 - .../pipeline-template/lib/NfcoreSchema.groovy | 1 - .../lib/NfcoreTemplate.groovy | 26 ++++++++++++++++--- .../pipeline-template/lib/WorkflowMain.groovy | 11 ++++++-- nf_core/pipeline-template/nextflow.config | 1 + .../pipeline-template/nextflow_schema.json | 6 +++++ 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccea47da4f..6c7dd4fba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Fix lint warnings for `samplesheet_check.nf` module - Add codespaces template ([#1957](https://github.com/nf-core/tools/pull/1957)) - Check that the workflow name provided with a template doesn't contain dashes ([#1822](https://github.com/nf-core/tools/pull/1822)) +- `nextflow run --version` will now print the workflow version from the manifest and exit ([#1951](https://github.com/nf-core/tools/pull/1951)) ### Linting diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index cf0fbf4d40..0cf98a5b39 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -141,7 +141,6 @@ def nextflow_config(self): ] # Old depreciated vars - fail if present config_fail_ifdefined = [ - "params.version", "params.nf_required_version", "params.container", "params.singleEnd", diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index b3d092f809..33cd4f6e8d 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -46,7 +46,6 @@ class NfcoreSchema { 'quiet', 'syslog', 'v', - 'version', // Options for `nextflow run` command 'ansi', diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 57b72d4e6d..172578fe5a 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -32,6 +32,25 @@ class NfcoreTemplate { } } + // + // Generate version string + // + public static String version(workflow) { + String version_string = "" + + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string + } + // // Construct and send completion email // @@ -61,7 +80,7 @@ class NfcoreTemplate { misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp def email_fields = [:] - email_fields['version'] = workflow.manifest.version + email_fields['version'] = NfcoreTemplate.version(workflow) email_fields['runName'] = workflow.runName email_fields['success'] = workflow.success email_fields['dateComplete'] = workflow.complete @@ -170,7 +189,7 @@ class NfcoreTemplate { misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp def msg_fields = [:] - msg_fields['version'] = workflow.manifest.version + msg_fields['version'] = NfcoreTemplate.version(workflow) msg_fields['runName'] = workflow.runName msg_fields['success'] = workflow.success msg_fields['dateComplete'] = workflow.complete @@ -300,6 +319,7 @@ class NfcoreTemplate { // public static String logo(workflow, monochrome_logs) { Map colors = logColours(monochrome_logs) + String workflow_version = NfcoreTemplate.version(workflow) String.format( """\n ${dashedLine(monochrome_logs)}{% if branded %} @@ -308,7 +328,7 @@ class NfcoreTemplate { ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} ${colors.green}`._,._,\'${colors.reset}{% endif %} - ${colors.purple} ${workflow.manifest.name} v${workflow.manifest.version}${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${workflow_version}${colors.reset} ${dashedLine(monochrome_logs)} """.stripIndent() ) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 11d956e9ec..32425c20e9 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -19,7 +19,7 @@ class WorkflowMain { } // - // Print help to screen if required + // Generate help string // public static String help(workflow, params, log) { {% if igenomes -%} @@ -36,7 +36,7 @@ class WorkflowMain { } // - // Print parameter summary log to screen + // Generate parameter summary log string // public static String paramsSummaryLog(workflow, params, log) { def summary_log = '' @@ -57,6 +57,13 @@ class WorkflowMain { System.exit(0) } + // Print workflow version and exit on --version + if (params.version) { + String workflow_version = NfcoreTemplate.version(workflow) + log.info "${workflow.manifest.name} ${workflow_version}" + System.exit(0) + } + // Validate workflow parameters via the JSON schema if (params.validate_params) { NfcoreSchema.validateParameters(workflow, params, log) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 99acd8a0af..52b3ec5c2e 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -37,6 +37,7 @@ params { monochrome_logs = false hook_url = null help = false + version = false validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 976f2719c8..72c4e163a9 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -176,6 +176,12 @@ "fa_icon": "fas fa-question-circle", "hidden": true }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, "publish_dir_mode": { "type": "string", "default": "copy", From e6428e5137cd0287316cf9374d6dd04d2bf96543 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 17 Nov 2022 12:56:42 +0100 Subject: [PATCH 560/854] modify install log.info messages and set command as a class --- nf_core/components/components_install.py | 103 --------- nf_core/components/install.py | 265 +++++++++++++++++++++++ nf_core/components/update.py | 5 +- nf_core/modules/install.py | 112 +--------- nf_core/subworkflows/install.py | 178 +-------------- 5 files changed, 291 insertions(+), 372 deletions(-) delete mode 100644 nf_core/components/components_install.py create mode 100644 nf_core/components/install.py diff --git a/nf_core/components/components_install.py b/nf_core/components/components_install.py deleted file mode 100644 index 5ad00695bc..0000000000 --- a/nf_core/components/components_install.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -import os - -import questionary - -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_utils import prompt_component_version_sha -from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME - -log = logging.getLogger(__name__) - - -def collect_and_verify_name(component_type, component, modules_repo): - """ - Collect component name. - Check that the supplied name is an available module/subworkflow. - """ - if component is None: - component = questionary.autocomplete( - f"{'Tool' if component_type == 'modules' else 'Subworkflow'} name:", - choices=modules_repo.get_avail_components(component_type), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Check that the supplied name is an available module/subworkflow - if component and component not in modules_repo.get_avail_components(component_type): - log.error(f"{component_type[:-1].title()} '{component}' not found in list of available {component_type}.") - log.info(f"Use the command 'nf-core {component_type} list' to view available software") - return False - - if not modules_repo.component_exists(component, component_type): - warn_msg = f"{component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" - log.warning(warn_msg) - return False - - return component - - -def check_component_installed(component_type, component, current_version, component_dir, modules_repo, force, prompt): - """ - Check that the module/subworkflow is not already installed - """ - if (current_version is not None and os.path.exists(component_dir)) and not force: - log.info(f"{component_type[:-1].title()} is already installed.") - - if prompt: - message = "?" if component_type == "modules" else " of this subworkflow and all it's imported modules?" - force = questionary.confirm( - f"{component_type[:-1].title()} {component} is already installed. \nDo you want to force the reinstallation{message}", - style=nf_core.utils.nfcore_question_style, - default=False, - ).unsafe_ask() - - if not force: - repo_flag = "" if modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {modules_repo.remote_url} " - branch_flag = "" if modules_repo.branch == "master" else f"-b {modules_repo.branch} " - - log.info( - f"To update '{component}' run 'nf-core {component_type} {repo_flag}{branch_flag}update {component}'. To force reinstallation use '--force'" - ) - return False - - return True - - -def get_version(component, component_type, sha, prompt, current_version, modules_repo): - """ - Get the version to install - """ - if sha: - version = sha - elif prompt: - try: - version = prompt_component_version_sha( - component, - component_type, - installed_sha=current_version, - modules_repo=modules_repo, - ) - except SystemError as e: - log.error(e) - return False - else: - # Fetch the latest commit for the module - version = modules_repo.get_latest_component_version(component, component_type) - return version - - -def clean_modules_json(component, component_type, modules_repo, modules_json): - """ - Remove installed version of module/subworkflow from modules.json - """ - for repo_url, repo_content in modules_json.modules_json["repos"].items(): - for dir, dir_components in repo_content[component_type].items(): - for name, component_values in dir_components.items(): - if name == component and dir == modules_repo.repo_path: - repo_to_remove = repo_url - log.info( - f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" - ) - modules_json.remove_entry(component_type, component, repo_to_remove, modules_repo.repo_path) - return component_values["installed_by"] diff --git a/nf_core/components/install.py b/nf_core/components/install.py new file mode 100644 index 0000000000..592299aa90 --- /dev/null +++ b/nf_core/components/install.py @@ -0,0 +1,265 @@ +import logging +import os +import re +from pathlib import Path + +import nf_core.modules.modules_utils +import nf_core.utils +import questionary +from nf_core.components.components_command import ComponentCommand +from nf_core.components.components_utils import prompt_component_version_sha +from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME + +log = logging.getLogger(__name__) + + +class ComponentInstall(ComponentCommand): + def __init__( + self, + pipeline_dir, + component_type, + force=False, + prompt=False, + sha=None, + remote_url=None, + branch=None, + no_pull=False, + installed_by=False, + ): + super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.force = force + self.prompt = prompt + self.sha = sha + if installed_by: + self.installed_by = installed_by + else: + self.installed_by = self.component_type + + def install(self, component, silent=False): + if self.repo_type == "modules": + log.error(f"You cannot install a {component} in a clone of nf-core/modules") + return False + # Check whether pipelines is valid + if not self.has_valid_directory(): + return False + + # Check modules directory structure + self.check_modules_structure() + + # Verify that 'modules.json' is consistent with the installed modules and subworkflows + modules_json = ModulesJson(self.dir) + modules_json.check_up_to_date() + + # Verify SHA + if not self.modules_repo.verify_sha(self.prompt, self.sha): + return False + + # Check and verify component name + component = self.collect_and_verify_name(component, self.modules_repo) + if not component: + return False + + # Get current version + current_version = modules_json.get_component_version( + self.component_type, component, self.modules_repo.remote_url, self.modules_repo.repo_path + ) + + # Set the install folder based on the repository name + install_folder = os.path.join(self.dir, self.component_type, self.modules_repo.repo_path) + + # Compute the component directory + component_dir = os.path.join(install_folder, component) + + # Check that the component is not already installed + component_installed = self.check_component_installed( + component, current_version, component_dir, self.modules_repo, self.force, self.prompt, silent + ) + if not component_installed: + log.debug( + f"{self.component_type[:-1].title()} is already installed and force is not set.\nAdding the new installation source {self.installed_by} for {self.component_type[:-1]} {component} to 'modules.json' without installing the {self.component_type}." + ) + modules_json.load() + modules_json.update(self.component_type, self.modules_repo, component, current_version, self.installed_by) + return False + + version = self.get_version(component, self.sha, self.prompt, current_version, self.modules_repo) + if not version: + return False + + # Remove component if force is set and component is installed + install_track = None + if self.force and component_installed: + log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{component}'") + self.clear_component_dir(component, component_dir) + install_track = self.clean_modules_json(component, self.modules_repo, modules_json) + + log.info(f"{'Rei' if self.force else 'I'}nstalling '{component}'") + log.debug( + f"Installing {self.component_type} '{component}' at modules hash {version} from {self.modules_repo.remote_url}" + ) + + # Download component files + if not self.install_component_files(component, version, self.modules_repo, install_folder): + return False + + # Update module.json with newly installed subworkflow + modules_json.load() + modules_json.update( + self.component_type, self.modules_repo, component, version, self.installed_by, install_track + ) + + if self.component_type == "subworkflows": + # Install included modules and subworkflows + self.install_included_components(component_dir) + + if not silent: + # Print include statement + component_name = "_".join(component.upper().split("/")) + log.info( + f"Include statement: include {{ {component_name} }} from '.{os.path.join(install_folder, component)}/main'" + ) + if self.component_type == "subworkflows": + subworkflow_config = os.path.join(install_folder, component, "nextflow.config") + if os.path.isfile(subworkflow_config): + log.info(f"Subworkflow config include statement: includeConfig '{subworkflow_config}'") + + return True + + def get_modules_subworkflows_to_install(self, subworkflow_dir): + """ + Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows. + """ + modules = [] + subworkflows = [] + with open(Path(subworkflow_dir, "main.nf"), "r") as fh: + for line in fh: + regex = re.compile( + r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" + ) + match = regex.match(line) + if match and len(match.groups()) == 2: + name, link = match.groups() + if link.startswith("../../../"): + name_split = name.lower().split("_") + modules.append("/".join(name_split)) + elif link.startswith("../"): + subworkflows.append(name.lower()) + return modules, subworkflows + + def install_included_components(self, subworkflow_dir): + """ + Install included modules and subworkflows + """ + modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) + for s_install in subworkflows_to_install: + original_installed = self.installed_by + self.installed_by = Path(subworkflow_dir).parts[-1] + self.install(s_install, silent=True) + self.installed_by = original_installed + for m_install in modules_to_install: + original_component_type = self.component_type + self.component_type = "modules" + original_installed = self.installed_by + self.installed_by = Path(subworkflow_dir).parts[-1] + self.install(m_install, silent=True) + self.component_type = original_component_type + self.installed_by = original_installed + + def collect_and_verify_name(self, component, modules_repo): + """ + Collect component name. + Check that the supplied name is an available module/subworkflow. + """ + if component is None: + component = questionary.autocomplete( + f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:", + choices=modules_repo.get_avail_components(self.component_type), + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Check that the supplied name is an available module/subworkflow + if component and component not in modules_repo.get_avail_components(self.component_type): + log.error( + f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." + ) + log.info(f"Use the command 'nf-core {self.component_type} list' to view available software") + return False + + if not modules_repo.component_exists(component, self.component_type): + warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" + log.warning(warn_msg) + return False + + return component + + def check_component_installed(self, component, current_version, component_dir, modules_repo, force, prompt, silent): + """ + Check that the module/subworkflow is not already installed + """ + if (current_version is not None and os.path.exists(component_dir)) and not force: + if not silent: + log.info(f"{self.component_type[:-1].title()} is already installed.") + + if prompt: + message = ( + "?" if self.component_type == "modules" else " of this subworkflow and all it's imported modules?" + ) + force = questionary.confirm( + f"{self.component_type[:-1].title()} {component} is already installed. \nDo you want to force the reinstallation{message}", + style=nf_core.utils.nfcore_question_style, + default=False, + ).unsafe_ask() + + if not force: + if not silent: + repo_flag = ( + "" if modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {modules_repo.remote_url} " + ) + branch_flag = "" if modules_repo.branch == "master" else f"-b {modules_repo.branch} " + + log.info( + f"To update '{component}' run 'nf-core {self.component_type} {repo_flag}{branch_flag}update {component}'. To force reinstallation use '--force'" + ) + return False + + return True + + def get_version(self, component, sha, prompt, current_version, modules_repo): + """ + Get the version to install + """ + if sha: + version = sha + elif prompt: + try: + version = prompt_component_version_sha( + component, + self.component_type, + installed_sha=current_version, + modules_repo=modules_repo, + ) + except SystemError as e: + log.error(e) + return False + else: + # Fetch the latest commit for the module + version = modules_repo.get_latest_component_version(component, self.component_type) + return version + + def clean_modules_json(self, component, modules_repo, modules_json): + """ + Remove installed version of module/subworkflow from modules.json + """ + for repo_url, repo_content in modules_json.modules_json["repos"].items(): + for dir, dir_components in repo_content[self.component_type].items(): + for name, component_values in dir_components.items(): + if name == component and dir == modules_repo.repo_path: + repo_to_remove = repo_url + log.info( + f"Removing {self.component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json" + ) + modules_json.remove_entry( + self.component_type, component, repo_to_remove, modules_repo.repo_path + ) + return component_values["installed_by"] diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 4e2025a158..38d0f306f1 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -4,10 +4,9 @@ import tempfile from pathlib import Path -import questionary - import nf_core.modules.modules_utils import nf_core.utils +import questionary from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_differ import ModulesDiffer @@ -248,7 +247,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr style=nf_core.utils.nfcore_question_style, ).unsafe_ask() if recursive_update and len(modules_to_update + subworkflows_to_update) > 0: - # Write all the differences of linked componenets to a diff file + # Write all the differences of linked components to a diff file self.update_linked_components( modules_to_update, subworkflows_to_update, updated, check_diff_exist=False ) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 346157d7a5..e1755cee98 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -1,16 +1,7 @@ -import logging -import os +from nf_core.components.install import ComponentInstall -import nf_core.components.components_install -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson -log = logging.getLogger(__name__) - - -class ModuleInstall(ComponentCommand): +class ModuleInstall(ComponentInstall): def __init__( self, pipeline_dir, @@ -22,93 +13,14 @@ def __init__( no_pull=False, installed_by=False, ): - super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) - self.force = force - self.prompt = prompt - self.sha = sha - if installed_by: - self.installed_by = installed_by - else: - self.installed_by = self.component_type - - def install(self, module, silent=False): - if self.repo_type == "modules": - log.error("You cannot install a module in a clone of nf-core/modules") - return False - # Check whether pipelines is valid - if not self.has_valid_directory(): - return False - - # Check modules directory structure - self.check_modules_structure() - - # Verify that 'modules.json' is consistent with the installed modules - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - - # Verify SHA - if not self.modules_repo.verify_sha(self.prompt, self.sha): - return False - - # Check and verify module name - module = nf_core.components.components_install.collect_and_verify_name( - self.component_type, module, self.modules_repo + super().__init__( + pipeline_dir, + "modules", + force=force, + prompt=prompt, + sha=sha, + remote_url=remote_url, + branch=branch, + no_pull=no_pull, + installed_by=installed_by, ) - if not module: - return False - - # Get current version - current_version = modules_json.get_module_version( - module, self.modules_repo.remote_url, self.modules_repo.repo_path - ) - - # Set the install folder based on the repository name - install_folder = os.path.join(self.dir, "modules", self.modules_repo.repo_path) - - # Compute the module directory - module_dir = os.path.join(install_folder, module) - - # Check that the module is not already installed - if not nf_core.components.components_install.check_component_installed( - self.component_type, module, current_version, module_dir, self.modules_repo, self.force, self.prompt - ): - log.debug( - f"Module is already installed and force is not set.\nAdding the new installation source {self.installed_by} for module {module} to 'modules.json' without installing the module." - ) - modules_json.load() - modules_json.update(self.component_type, self.modules_repo, module, current_version, self.installed_by) - return False - - version = nf_core.components.components_install.get_version( - module, self.component_type, self.sha, self.prompt, current_version, self.modules_repo - ) - if not version: - return False - - # Remove module if force is set - install_track = None - if self.force: - log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'") - self.clear_component_dir(module, module_dir) - install_track = nf_core.components.components_install.clean_modules_json( - module, self.component_type, self.modules_repo, modules_json - ) - - log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'") - log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}") - - # Download module files - if not self.install_component_files(module, version, self.modules_repo, install_folder): - return False - - if not silent: - # Print include statement - module_name = "_".join(module.upper().split("/")) - log.info( - f"Include statement: include {{ {module_name} }} from '.{os.path.join(install_folder, module)}/main'" - ) - - # Update module.json with newly installed module - modules_json.load() - modules_json.update(self.component_type, self.modules_repo, module, version, self.installed_by, install_track) - return True diff --git a/nf_core/subworkflows/install.py b/nf_core/subworkflows/install.py index d16e37e6c1..6c5cfb12b2 100644 --- a/nf_core/subworkflows/install.py +++ b/nf_core/subworkflows/install.py @@ -1,19 +1,7 @@ -import logging -import os -import re -from pathlib import Path +from nf_core.components.install import ComponentInstall -import nf_core.components.components_install -import nf_core.modules.modules_utils -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.install import ModuleInstall -from nf_core.modules.modules_json import ModulesJson -log = logging.getLogger(__name__) - - -class SubworkflowInstall(ComponentCommand): +class SubworkflowInstall(ComponentInstall): def __init__( self, pipeline_dir, @@ -25,156 +13,14 @@ def __init__( no_pull=False, installed_by=False, ): - super().__init__("subworkflows", pipeline_dir, remote_url, branch, no_pull) - self.force = force - self.prompt = prompt - self.sha = sha - if installed_by: - self.installed_by = installed_by - else: - self.installed_by = self.component_type - - def install(self, subworkflow, silent=False): - if self.repo_type == "modules": - log.error("You cannot install a subworkflow in a clone of nf-core/modules") - return False - # Check whether pipelines is valid - if not self.has_valid_directory(): - return False - - # Verify that 'modules.json' is consistent with the installed modules and subworkflows - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - - # Verify SHA - if not self.modules_repo.verify_sha(self.prompt, self.sha): - return False - - # Check and verify subworkflow name - subworkflow = nf_core.components.components_install.collect_and_verify_name( - self.component_type, subworkflow, self.modules_repo + super().__init__( + pipeline_dir, + "subworkflows", + force=force, + prompt=prompt, + sha=sha, + remote_url=remote_url, + branch=branch, + no_pull=no_pull, + installed_by=installed_by, ) - if not subworkflow: - return False - - # Get current version - current_version = modules_json.get_subworkflow_version( - subworkflow, self.modules_repo.remote_url, self.modules_repo.repo_path - ) - - # Set the install folder based on the repository name - install_folder = os.path.join(self.dir, "subworkflows", self.modules_repo.repo_path) - - # Compute the subworkflow directory - subworkflow_dir = os.path.join(install_folder, subworkflow) - - # Check that the subworkflow is not already installed - if not nf_core.components.components_install.check_component_installed( - self.component_type, - subworkflow, - current_version, - subworkflow_dir, - self.modules_repo, - self.force, - self.prompt, - ): - log.debug( - f"Subworkflow is already installed and force is not set.\nAdding the new installation source {self.installed_by} for subworkflow {subworkflow} to 'modules.json' without installing the subworkflow." - ) - modules_json.load() - modules_json.update(self.component_type, self.modules_repo, subworkflow, current_version, self.installed_by) - return False - - version = nf_core.components.components_install.get_version( - subworkflow, self.component_type, self.sha, self.prompt, current_version, self.modules_repo - ) - if not version: - return False - - # Remove subworkflow if force is set and component is installed - install_track = None - if self.force and nf_core.components.components_install.check_component_installed( - self.component_type, - subworkflow, - current_version, - subworkflow_dir, - self.modules_repo, - self.force, - self.prompt, - ): - log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{subworkflow}'") - self.clear_component_dir(subworkflow, subworkflow_dir) - install_track = nf_core.components.components_install.clean_modules_json( - subworkflow, self.component_type, self.modules_repo, modules_json - ) - - log.info(f"{'Rei' if self.force else 'I'}nstalling '{subworkflow}'") - log.debug(f"Installing subworkflow '{subworkflow}' at hash {version} from {self.modules_repo.remote_url}") - - # Download subworkflow files - if not self.install_component_files(subworkflow, version, self.modules_repo, install_folder): - return False - - # Update module.json with newly installed subworkflow - modules_json.load() - modules_json.update( - self.component_type, self.modules_repo, subworkflow, version, self.installed_by, install_track - ) - - # Install included modules and subworkflows - self.install_included_components(subworkflow_dir) - - if not silent: - # Print include statement - subworkflow_name = subworkflow.upper() - log.info( - f"Include statement: include {{ {subworkflow_name} }} from '.{os.path.join(install_folder, subworkflow)}/main'" - ) - subworkflow_config = os.path.join(install_folder, subworkflow, "nextflow.config") - if os.path.isfile(subworkflow_config): - log.info(f"Subworkflow config include statement: includeConfig '{subworkflow_config}'") - - return True - - def get_modules_subworkflows_to_install(self, subworkflow_dir): - """ - Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows. - """ - modules = [] - subworkflows = [] - with open(Path(subworkflow_dir, "main.nf"), "r") as fh: - for line in fh: - regex = re.compile( - r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" - ) - match = regex.match(line) - if match and len(match.groups()) == 2: - name, link = match.groups() - if link.startswith("../../../"): - name_split = name.lower().split("_") - modules.append("/".join(name_split)) - elif link.startswith("../"): - subworkflows.append(name.lower()) - return modules, subworkflows - - def install_included_components(self, subworkflow_dir): - """ - Install included modules and subworkflows - """ - modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir) - for s_install in subworkflows_to_install: - original_installed = self.installed_by - self.installed_by = Path(subworkflow_dir).parts[-1] - self.install(s_install, silent=True) - self.installed_by = original_installed - for m_install in modules_to_install: - module_install = ModuleInstall( - self.dir, - force=self.force, - prompt=self.prompt, - sha=self.sha, - remote_url=self.modules_repo.remote_url, - branch=self.modules_repo.branch, - installed_by=Path(subworkflow_dir).parts[-1], - ) - module_install.install(m_install, silent=True) From 53ccaef64fdb5a9c6ac0b6b8419fa3034c34d5b0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 13:07:52 +0100 Subject: [PATCH 561/854] convert info to components, add tests --- nf_core/components/info.py | 295 ++++++++++++++++++++++++++++++++ nf_core/modules/info.py | 281 ++---------------------------- nf_core/modules/modules_repo.py | 29 +--- nf_core/subworkflows/info.py | 294 ++----------------------------- tests/modules/info.py | 49 ++++++ tests/subworkflows/info.py | 49 ++++++ tests/test_modules.py | 5 + tests/test_subworkflows.py | 5 + 8 files changed, 433 insertions(+), 574 deletions(-) create mode 100644 nf_core/components/info.py create mode 100644 tests/modules/info.py create mode 100644 tests/subworkflows/info.py diff --git a/nf_core/components/info.py b/nf_core/components/info.py new file mode 100644 index 0000000000..cfa71cc842 --- /dev/null +++ b/nf_core/components/info.py @@ -0,0 +1,295 @@ +import logging +import os +from pathlib import Path + +import questionary +import yaml +from rich import box +from rich.console import Group +from rich.markdown import Markdown +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +import nf_core.utils +from nf_core.components.components_command import ComponentCommand +from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE +from nf_core.modules.modules_utils import get_repo_type + +log = logging.getLogger(__name__) + + +class ComponentInfo(ComponentCommand): + """ + Class to print information of a module/subworkflow. + + Attributes + ---------- + meta : YAML object + stores the information from meta.yml file + local_path : str + path of the local modules/subworkflows + remote_location : str + remote repository URL + local : bool + indicates if the module/subworkflow is locally installed or not + repo_type : str + repository type. Can be either 'pipeline' or 'modules' + modules_json : ModulesJson object + contains 'modules.json' file information from a pipeline + component_name : str + name of the module/subworkflow to get information from + + Methods + ------- + init_mod_name(component) + Makes sure that we have a modules/subworkflows name + get_component_info() + Given the name of a module/subworkflow, parse meta.yml and print usage help + get_local_yaml() + Attempt to get the meta.yml file from a locally installed module/subworkflow + get_remote_yaml() + Attempt to get the meta.yml file from a remote repo + generate_component_info_help() + Take the parsed meta.yml and generate rich help + """ + + def __init__( + self, + component_type, + pipeline_dir, + component_name, + remote_url=None, + branch=None, + no_pull=False, + ): + super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.meta = None + self.local_path = None + self.remote_location = None + self.local = None + + # Quietly check if this is a pipeline or not + if pipeline_dir: + try: + pipeline_dir, repo_type = get_repo_type(pipeline_dir, use_prompt=False) + log.debug(f"Found {repo_type} repo: {pipeline_dir}") + except UserWarning as e: + log.debug(f"Only showing remote info: {e}") + pipeline_dir = None + + if self.repo_type == "pipeline": + # Check modules directory structure + if self.component_type == "modules": + self.check_modules_structure() + # Check modules.json up to date + self.modules_json = ModulesJson(self.dir) + self.modules_json.check_up_to_date() + else: + self.modules_json = None + self.component = self.init_mod_name(component_name) + + def init_mod_name(self, component): + """ + Makes sure that we have a module/subworkflow name before proceeding. + + Args: + module: str: Module name to check + """ + if component is None: + self.local = questionary.confirm( + f"Is the {self.component_type[:-1]} locally installed?", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + if self.local: + if self.repo_type == "modules": + components = self.get_components_clone_modules() + else: + components = self.modules_json.get_all_components(self.component_type).get( + self.modules_repo.remote_url + ) + components = [ + component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components + ] + if components is None: + raise UserWarning( + f"No {self.component_type[:-1]} installed from '{self.modules_repo.remote_url}'" + ) + else: + components = self.modules_repo.get_avail_components(self.component_type) + module = questionary.autocomplete( + "Please select a {self.component_type[:-1]}", + choices=components, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + while component not in components: + log.info(f"'{component}' is not a valid {self.component_type[:-1]} name") + component = questionary.autocomplete( + f"Please select a new {self.component_type[:-1]}", + choices=components, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + return component + + def get_component_info(self): + """Given the name of a module/subworkflow, parse meta.yml and print usage help.""" + + # Running with a local install, try to find the local meta + if self.local: + self.meta = self.get_local_yaml() + + # Either failed locally or in remote mode + if not self.meta: + self.meta = self.get_remote_yaml() + + # Could not find the meta + if self.meta is False: + raise UserWarning(f"Could not find {self.component_type[:-1]} '{self.component}'") + + return self.generate_component_info_help() + + def get_local_yaml(self): + """Attempt to get the meta.yml file from a locally installed module/subworkflow. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + + if self.repo_type == "pipeline": + # Try to find and load the meta.yml file + component_base_path = Path(self.dir, self.component_type) + # Check that we have any modules/subworkflows installed from this repo + components = self.modules_json.get_all_components(self.component_type).get(self.modules_repo.remote_url) + component_names = [component for _, component in components] + if components is None: + raise LookupError(f"No {self.component_type[:-1]} installed from {self.modules_repo.remote_url}") + + if self.component in component_names: + install_dir = [dir for dir, module in components if module == self.component][0] + comp_dir = Path(component_base_path, install_dir, self.component) + meta_fn = Path(comp_dir, "meta.yml") + if meta_fn.exists(): + log.debug(f"Found local file: {meta_fn}") + with open(meta_fn, "r") as fh: + self.local_path = comp_dir + return yaml.safe_load(fh) + + log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally") + else: + component_base_path = Path(self.dir, self.component_type, "nf-core") + if self.component in os.listdir(component_base_path): + comp_dir = Path(component_base_path, self.component) + meta_fn = Path(comp_dir, "meta.yml") + if meta_fn.exists(): + log.debug(f"Found local file: {meta_fn}") + with open(meta_fn, "r") as fh: + self.local_path = comp_dir + return yaml.safe_load(fh) + log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally") + + return None + + def get_remote_yaml(self): + """Attempt to get the meta.yml file from a remote repo. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + # Check if our requested module/subworkflow is there + if self.component not in self.modules_repo.get_avail_components(self.component_type): + return False + + file_contents = self.modules_repo.get_meta_yml(self.component_type, self.component) + if file_contents is None: + return False + self.remote_location = self.modules_repo.remote_url + return yaml.safe_load(file_contents) + + def generate_component_info_help(self): + """Take the parsed meta.yml and generate rich help. + + Returns: + rich renderable + """ + + renderables = [] + + # Intro panel + intro_text = Text() + if self.local_path: + intro_text.append(Text.from_markup(f"Location: [blue]{self.local_path}\n")) + elif self.remote_location: + intro_text.append( + Text.from_markup( + ":globe_with_meridians: Repository: " + f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" + f"{self.remote_location}" + f"{'[/link]' if self.remote_location.startswith('http') else '' }" + "\n" + ) + ) + + if self.meta.get("tools"): + tools_strings = [] + for tool in self.meta["tools"]: + for tool_name, tool_meta in tool.items(): + if "homepage" in tool_meta: + tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}[/link]") + else: + tools_strings.append(f"{tool_name}") + intro_text.append(Text.from_markup(f":wrench: Tools: {', '.join(tools_strings)}\n", style="dim")) + + if self.meta.get("description"): + intro_text.append(Text.from_markup(f":book: Description: {self.meta['description']}", style="dim")) + + renderables.append( + Panel( + intro_text, + title=f"[bold]{self.component_type[:-1].title()}: [green]{self.component}\n", + title_align="left", + ) + ) + + # Inputs + if self.meta.get("input"): + inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + inputs_table.add_column(":inbox_tray: Inputs") + inputs_table.add_column("Description") + inputs_table.add_column("Pattern", justify="right", style="green") + for input in self.meta["input"]: + for key, info in input.items(): + inputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + + renderables.append(inputs_table) + + # Outputs + if self.meta.get("output"): + outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + outputs_table.add_column(":outbox_tray: Outputs") + outputs_table.add_column("Description") + outputs_table.add_column("Pattern", justify="right", style="green") + for output in self.meta["output"]: + for key, info in output.items(): + outputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + + renderables.append(outputs_table) + + # Installation command + if self.remote_location: + cmd_base = f"nf-core {self.component_type}" + if self.remote_location != NF_CORE_MODULES_REMOTE: + cmd_base = f"nf-core {self.component_type} --git-remote {self.remote_location}" + renderables.append( + Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.component}\n") + ) + + return Group(*renderables) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 2db3891ce0..265bbb2678 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -1,276 +1,17 @@ import logging -import os -import questionary -import yaml -from rich import box -from rich.console import Group -from rich.markdown import Markdown -from rich.panel import Panel -from rich.table import Table -from rich.text import Text - -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson - -from .modules_repo import NF_CORE_MODULES_REMOTE -from .modules_utils import get_repo_type +from nf_core.components.info import ComponentInfo log = logging.getLogger(__name__) -class ModuleInfo(ComponentCommand): - """ - Class to print information of a module. - - Attributes - ---------- - meta : YAML object - stores the information from meta.yml file - local_path : str - path of the local modules - remote_location : str - remote repository URL - local : bool - indicates if the module is locally installed or not - repo_type : str - repository type. Can be either 'pipeline' or 'modules' - modules_json : ModulesJson object - contains 'modules.json' file information from a pipeline - module : str - name of the tool to get information from - - Methods - ------- - init_mod_name(module) - Makes sure that we have a module name - get_module_info() - Given the name of a module, parse meta.yml and print usage help - get_local_yaml() - Attempt to get the meta.yml file from a locally installed module - get_remote_yaml() - Attempt to get the meta.yml file from a remote repo - generate_module_info_help() - Take the parsed meta.yml and generate rich help - """ - - def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): - super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) - self.meta = None - self.local_path = None - self.remote_location = None - self.local = None - - # Quietly check if this is a pipeline or not - if pipeline_dir: - try: - pipeline_dir, repo_type = get_repo_type(pipeline_dir, use_prompt=False) - log.debug(f"Found {repo_type} repo: {pipeline_dir}") - except UserWarning as e: - log.debug(f"Only showing remote info: {e}") - pipeline_dir = None - - if self.repo_type == "pipeline": - # Check modules directory structure - self.check_modules_structure() - # Check modules.json up to date - self.modules_json = ModulesJson(self.dir) - self.modules_json.check_up_to_date() - else: - self.modules_json = None - self.module = self.init_mod_name(tool) - - def init_mod_name(self, module): - """ - Makes sure that we have a module name before proceeding. - - Args: - module: str: Module name to check - """ - if module is None: - self.local = questionary.confirm( - "Is the module locally installed?", style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - if self.local: - if self.repo_type == "modules": - modules = self.get_components_clone_modules() - else: - modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) - modules = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules] - if modules is None: - raise UserWarning(f"No modules installed from '{self.modules_repo.remote_url}'") - else: - modules = self.modules_repo.get_avail_components(self.component_type) - module = questionary.autocomplete( - "Please select a module", choices=modules, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - while module not in modules: - log.info(f"'{module}' is not a valid module name") - module = questionary.autocomplete( - "Please select a new module", choices=modules, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - - return module - - def get_module_info(self): - """Given the name of a module, parse meta.yml and print usage help.""" - - # Running with a local install, try to find the local meta - if self.local: - self.meta = self.get_local_yaml() - - # Either failed locally or in remote mode - if not self.meta: - self.meta = self.get_remote_yaml() - - # Could not find the meta - if self.meta is False: - raise UserWarning(f"Could not find module '{self.module}'") - - return self.generate_module_info_help() - - def get_local_yaml(self): - """Attempt to get the meta.yml file from a locally installed module. - - Returns: - dict or bool: Parsed meta.yml found, False otherwise - """ - - if self.repo_type == "pipeline": - # Try to find and load the meta.yml file - module_base_path = os.path.join(self.dir, "modules") - # Check that we have any modules installed from this repo - modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url) - module_names = [module for _, module in modules] - if modules is None: - raise LookupError(f"No modules installed from {self.modules_repo.remote_url}") - - if self.module in module_names: - install_dir = [dir for dir, module in modules if module == self.module][0] - mod_dir = os.path.join(module_base_path, install_dir, self.module) - meta_fn = os.path.join(mod_dir, "meta.yml") - if os.path.exists(meta_fn): - log.debug(f"Found local file: {meta_fn}") - with open(meta_fn, "r") as fh: - self.local_path = mod_dir - return yaml.safe_load(fh) - - log.debug(f"Module '{self.module}' meta.yml not found locally") - else: - module_base_path = os.path.join(self.dir, "modules", "nf-core") - if self.module in os.listdir(module_base_path): - mod_dir = os.path.join(module_base_path, self.module) - meta_fn = os.path.join(mod_dir, "meta.yml") - if os.path.exists(meta_fn): - log.debug(f"Found local file: {meta_fn}") - with open(meta_fn, "r") as fh: - self.local_path = mod_dir - return yaml.safe_load(fh) - log.debug(f"Module '{self.module}' meta.yml not found locally") - - return None - - def get_remote_yaml(self): - """Attempt to get the meta.yml file from a remote repo. - - Returns: - dict or bool: Parsed meta.yml found, False otherwise - """ - # Check if our requested module is there - if self.module not in self.modules_repo.get_avail_components(self.component_type): - return False - - file_contents = self.modules_repo.get_meta_yml(self.module) - if file_contents is None: - return False - self.remote_location = self.modules_repo.remote_url - return yaml.safe_load(file_contents) - - def generate_module_info_help(self): - """Take the parsed meta.yml and generate rich help. - - Returns: - rich renderable - """ - - renderables = [] - - # Intro panel - intro_text = Text() - if self.local_path: - intro_text.append(Text.from_markup(f"Location: [blue]{self.local_path}\n")) - elif self.remote_location: - intro_text.append( - Text.from_markup( - ":globe_with_meridians: Repository: " - f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" - f"{self.remote_location}" - f"{'[/link]' if self.remote_location.startswith('http') else '' }" - "\n" - ) - ) - - if self.meta.get("tools"): - tools_strings = [] - for tool in self.meta["tools"]: - for tool_name, tool_meta in tool.items(): - if "homepage" in tool_meta: - tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}[/link]") - else: - tools_strings.append(f"{tool_name}") - intro_text.append(Text.from_markup(f":wrench: Tools: {', '.join(tools_strings)}\n", style="dim")) - - if self.meta.get("description"): - intro_text.append(Text.from_markup(f":book: Description: {self.meta['description']}", style="dim")) - - renderables.append( - Panel( - intro_text, - title=f"[bold]Module: [green]{self.module}\n", - title_align="left", - ) - ) - - # Inputs - if self.meta.get("input"): - inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) - inputs_table.add_column(":inbox_tray: Inputs") - inputs_table.add_column("Description") - inputs_table.add_column("Pattern", justify="right", style="green") - for input in self.meta["input"]: - for key, info in input.items(): - inputs_table.add_row( - f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"] if info["description"] else ""), - info.get("pattern", ""), - ) - - renderables.append(inputs_table) - - # Outputs - if self.meta.get("output"): - outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) - outputs_table.add_column(":outbox_tray: Outputs") - outputs_table.add_column("Description") - outputs_table.add_column("Pattern", justify="right", style="green") - for output in self.meta["output"]: - for key, info in output.items(): - outputs_table.add_row( - f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"] if info["description"] else ""), - info.get("pattern", ""), - ) - - renderables.append(outputs_table) - - # Installation command - if self.remote_location: - cmd_base = "nf-core modules" - if self.remote_location != NF_CORE_MODULES_REMOTE: - cmd_base = f"nf-core modules --git-remote {self.remote_location}" - renderables.append( - Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.module}\n") - ) - - return Group(*renderables) +class ModuleInfo(ComponentInfo): + def __init__( + self, + pipeline_dir, + component_name, + remote_url=None, + branch=None, + no_pull=False, + ): + super().__init__("modules", pipeline_dir, component_name, remote_url, branch, no_pull) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index c268c8d8d2..673db73155 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -468,7 +468,7 @@ def get_avail_components(self, component_type, checkout=True): ] return avail_component_names - def get_meta_yml(self, module_name): + def get_meta_yml(self, component_type, module_name): """ Returns the contents of the 'meta.yml' file of a module @@ -479,26 +479,13 @@ def get_meta_yml(self, module_name): (str): The contents of the file in text format """ self.checkout_branch() - path = os.path.join(self.modules_dir, module_name, "meta.yml") - if not os.path.exists(path): - return None - with open(path) as fh: - contents = fh.read() - return contents - - def get_subworkflow_meta_yml(self, subworkflow_name): - """ - Returns the contents of the 'meta.yml' file of a subworkflow - - Args: - subworkflow_name (str): The name of the subworkflow - - Returns: - (str): The contents of the file in text format - """ - self.checkout_branch() - path = os.path.join(self.subworkflows_dir, subworkflow_name, "meta.yml") - if not os.path.exists(path): + if component_type == "modules": + path = Path(self.modules_dir, module_name, "meta.yml") + elif component_type == "subworkflows": + path = Path(self.subworkflows_dir, module_name, "meta.yml") + else: + raise ValueError(f"Invalid component type: {component_type}") + if not path.exists(): return None with open(path) as fh: contents = fh.read() diff --git a/nf_core/subworkflows/info.py b/nf_core/subworkflows/info.py index a428ffd311..48d78d41e8 100644 --- a/nf_core/subworkflows/info.py +++ b/nf_core/subworkflows/info.py @@ -1,289 +1,17 @@ import logging -import os -from pathlib import Path -import questionary -import yaml -from rich import box -from rich.console import Group -from rich.markdown import Markdown -from rich.panel import Panel -from rich.table import Table -from rich.text import Text - -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE, ModulesRepo -from nf_core.modules.modules_utils import get_repo_type +from nf_core.components.info import ComponentInfo log = logging.getLogger(__name__) -class SubworkflowInfo(ComponentCommand): - """ - Class to print information of a subworkflow. - - Attributes - ---------- - meta : YAML object - stores the information from meta.yml file - local_path : str - path of the local subworkflows - remote_location : str - remote repository URL - local : bool - indicates if the subworkflow is locally installed or not - repo_type : str - repository type. Can be either 'pipeline' or 'modules' - modules_json : ModulesJson object - contains 'modules.json' file information from a pipeline - module : str - name of the tool to get information from - - Methods - ------- - init_mod_name(subworkflow) - Makes sure that we have a subworkflow name - get_subworkflow_info() - Given the name of a subworkflow, parse meta.yml and print usage help - get_local_yaml() - Attempt to get the meta.yml file from a locally installed subworkflow - get_remote_yaml() - Attempt to get the meta.yml file from a remote repo - generate_subworkflow_info_help() - Take the parsed meta.yml and generate rich help - """ - - def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull): - super().__init__("subworkflows", pipeline_dir, remote_url, branch, no_pull) - self.meta = None - self.local_path = None - self.remote_location = None - self.local = None - - # Quietly check if this is a pipeline or not - if pipeline_dir: - try: - pipeline_dir, repo_type = get_repo_type(pipeline_dir, use_prompt=False) - log.debug(f"Found {repo_type} repo: {pipeline_dir}") - except UserWarning as e: - log.debug(f"Only showing remote info: {e}") - pipeline_dir = None - - if self.repo_type == "pipeline": - # # Check modules directory structure - # self.check_modules_structure() - # Check modules.json up to date - self.modules_json = ModulesJson(self.dir) - self.modules_json.check_up_to_date() - else: - self.modules_json = None - self.subworkflow = self.init_mod_name(tool) - - def init_mod_name(self, subworkflow): - """ - Makes sure that we have a subworkflow name before proceeding. - - Args: - subworkflow: str: Subworkflow name to check - """ - if subworkflow is None: - self.local = questionary.confirm( - "Is the subworkflow locally installed?", style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - if self.local: - if self.repo_type == "modules": - subworkflows = self.get_subworkflows_clone_modules() - else: - subworkflows = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - subworkflows = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in subworkflows] - if subworkflows is None: - raise UserWarning(f"No subworkflow installed from '{self.modules_repo.remote_url}'") - else: - subworkflows = self.modules_repo.get_avail_components(self.component_type) - subworkflow = questionary.autocomplete( - "Please select a subworkflow", choices=subworkflows, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - while subworkflow not in subworkflows: - log.info(f"'{subworkflow}' is not a valid subworkflow name") - subworkflow = questionary.autocomplete( - "Please select a new subworkflow", choices=subworkflows, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - - return subworkflow - - def get_subworkflow_info(self): - """Given the name of a subworkflow, parse meta.yml and print usage help.""" - - # Running with a local install, try to find the local meta - if self.local: - self.meta = self.get_local_yaml() - - # Either failed locally or in remote mode - if not self.meta: - self.meta = self.get_remote_yaml() - - # Could not find the meta - if self.meta is False: - raise UserWarning(f"Could not find subworkflow '{self.subworkflow}'") - - return self.generate_subworkflow_info_help() - - def get_local_yaml(self): - """Attempt to get the meta.yml file from a locally installed module. - - Returns: - dict or bool: Parsed meta.yml found, False otherwise - """ - - if self.repo_type == "pipeline": - # Try to find and load the meta.yml file - module_base_path = os.path.join(self.dir, "subworkflows") - # Check that we have any subworkflows installed from this repo - subworkflows = self.modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url) - subworkflow_names = [subworkflow for _, subworkflow in subworkflows] - if subworkflows is None: - raise LookupError(f"No subworkflows installed from {self.modules_repo.remote_url}") - - if self.subworkflow in subworkflow_names: - install_dir = [dir for dir, subworkflow in subworkflows if subworkflow == self.subworkflow][0] - mod_dir = os.path.join(module_base_path, install_dir, self.subworkflow) - meta_fn = os.path.join(mod_dir, "meta.yml") - if os.path.exists(meta_fn): - log.debug(f"Found local file: {meta_fn}") - with open(meta_fn, "r") as fh: - self.local_path = mod_dir - return yaml.safe_load(fh) - - log.debug(f"Subworkflow '{self.subworkflow}' meta.yml not found locally") - else: - module_base_path = os.path.join(self.dir, "subworkflows", "nf-core") - if self.subworkflow in os.listdir(module_base_path): - mod_dir = os.path.join(module_base_path, self.subworkflow) - meta_fn = os.path.join(mod_dir, "meta.yml") - if os.path.exists(meta_fn): - log.debug(f"Found local file: {meta_fn}") - with open(meta_fn, "r") as fh: - self.local_path = mod_dir - return yaml.safe_load(fh) - log.debug(f"Subworkflow '{self.subworkflow}' meta.yml not found locally") - - return None - - def get_remote_yaml(self): - """Attempt to get the meta.yml file from a remote repo. - - Returns: - dict or bool: Parsed meta.yml found, False otherwise - """ - # Check if our requested module is there - if self.subworkflow not in self.modules_repo.get_avail_components(self.component_type): - return False - - file_contents = self.modules_repo.get_subworkflow_meta_yml(self.subworkflow) - if file_contents is None: - return False - self.remote_location = self.modules_repo.remote_url - return yaml.safe_load(file_contents) - - def generate_subworkflow_info_help(self): - """Take the parsed meta.yml and generate rich help. - - Returns: - rich renderable - """ - - renderables = [] - - # Intro panel - intro_text = Text() - if self.local_path: - intro_text.append(Text.from_markup(f"Location: [blue]{self.local_path}\n")) - elif self.remote_location: - intro_text.append( - Text.from_markup( - ":globe_with_meridians: Repository: " - f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" - f"{self.remote_location}" - f"{'[/link]' if self.remote_location.startswith('http') else '' }" - "\n" - ) - ) - - if self.meta.get("tools"): - tools_strings = [] - for tool in self.meta["tools"]: - for tool_name, tool_meta in tool.items(): - if "homepage" in tool_meta: - tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}[/link]") - else: - tools_strings.append(f"{tool_name}") - intro_text.append(Text.from_markup(f":wrench: Tools: {', '.join(tools_strings)}\n", style="dim")) - - if self.meta.get("description"): - intro_text.append(Text.from_markup(f":book: Description: {self.meta['description']}", style="dim")) - - renderables.append( - Panel( - intro_text, - title=f"[bold]Subworkflow: [green]{self.subworkflow}\n", - title_align="left", - ) - ) - - # Inputs - if self.meta.get("input"): - inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) - inputs_table.add_column(":inbox_tray: Inputs") - inputs_table.add_column("Description") - inputs_table.add_column("Pattern", justify="right", style="green") - for input in self.meta["input"]: - for key, info in input.items(): - inputs_table.add_row( - f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"] if info["description"] else ""), - info.get("pattern", ""), - ) - - renderables.append(inputs_table) - - # Outputs - if self.meta.get("output"): - outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) - outputs_table.add_column(":outbox_tray: Outputs") - outputs_table.add_column("Description") - outputs_table.add_column("Pattern", justify="right", style="green") - for output in self.meta["output"]: - for key, info in output.items(): - outputs_table.add_row( - f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"] if info["description"] else ""), - info.get("pattern", ""), - ) - - renderables.append(outputs_table) - - # Installation command - if self.remote_location: - cmd_base = "nf-core subworkflows" - if self.remote_location != NF_CORE_MODULES_REMOTE: - cmd_base = f"nf-core subworkflows --git-remote {self.remote_location}" - renderables.append( - Text.from_markup( - f"\n :computer: Installation command: [magenta]{cmd_base} install {self.subworkflow}\n" - ) - ) - - return Group(*renderables) - - def get_subworkflows_clone_modules(self): - """ - Get the subworkflows available in a clone of nf-core/modules - """ - module_base_path = Path(self.dir, Path("subworkflows", "nf-core")) - return [ - str(Path(dir).relative_to(module_base_path)) - for dir, _, files in os.walk(module_base_path) - if "main.nf" in files - ] +class SubworkflowInfo(ComponentInfo): + def __init__( + self, + pipeline_dir, + component_name, + remote_url=None, + branch=None, + no_pull=False, + ): + super().__init__("subworkflows", pipeline_dir, component_name, remote_url, branch, no_pull) diff --git a/tests/modules/info.py b/tests/modules/info.py new file mode 100644 index 0000000000..58d90a6afc --- /dev/null +++ b/tests/modules/info.py @@ -0,0 +1,49 @@ +from rich.console import Console + +import nf_core.modules + +from ..utils import GITLAB_DEFAULT_BRANCH, GITLAB_URL + + +def test_modules_info_remote(self): + """Test getting info about a remote module""" + mods_info = nf_core.modules.ModuleInfo(self.pipeline_dir, "fastqc") + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Module: fastqc" in output + assert "Inputs" in output + assert "Outputs" in output + + +def test_modules_info_remote_gitlab(self): + """Test getting info about a module in the remote gitlab repo""" + mods_info = nf_core.modules.ModuleInfo( + self.pipeline_dir, "fastqc", remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH + ) + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Module: fastqc" in output + assert "Inputs" in output + assert "Outputs" in output + assert "--git-remote" in output + + +def test_modules_info_local(self): + """Test getting info about a locally installed module""" + self.mods_install.install("trimgalore") + mods_info = nf_core.modules.ModuleInfo(self.pipeline_dir, "trimgalore") + mods_info.local = True + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Module: trimgalore" in output + assert "Inputs" in output + assert "Outputs" in output diff --git a/tests/subworkflows/info.py b/tests/subworkflows/info.py new file mode 100644 index 0000000000..cda3d96f92 --- /dev/null +++ b/tests/subworkflows/info.py @@ -0,0 +1,49 @@ +from rich.console import Console + +import nf_core.subworkflows + +from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL + + +def test_subworkflows_info_remote(self): + """Test getting info about a remote subworkflow""" + mods_info = nf_core.subworkflows.SubworkflowInfo(self.pipeline_dir, "bam_sort_stats_samtools") + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Subworkflow: bam_sort_stats_samtools" in output + assert "Inputs" in output + assert "Outputs" in output + + +def test_subworkflows_info_remote_gitlab(self): + """Test getting info about a subworkflow in the remote gitlab repo""" + mods_info = nf_core.subworkflows.SubworkflowInfo( + self.pipeline_dir, "bam_sort_stats_samtools", remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH + ) + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Subworkflow: bam_sort_stats_samtools" in output + assert "Inputs" in output + assert "Outputs" in output + assert "--git-remote" in output + + +def test_subworkflows_info_local(self): + """Test getting info about a locally installed subworkflow""" + self.subworkflow_install.install("bam_sort_stats_samtools") + mods_info = nf_core.subworkflows.SubworkflowInfo(self.pipeline_dir, "bam_sort_stats_samtools") + mods_info.local = True + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Subworkflow: bam_sort_stats_samtools" in output + assert "Inputs" in output + assert "Outputs" in output diff --git a/tests/test_modules.py b/tests/test_modules.py index 7d5288d781..9c3e1bf2c2 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -135,6 +135,11 @@ def test_modulesrepo_class(self): test_modules_custom_yml_dumper, test_modules_test_file_dict, ) + from .modules.info import ( + test_modules_info_local, + test_modules_info_remote, + test_modules_info_remote_gitlab, + ) from .modules.install import ( test_modules_install_different_branch_fail, test_modules_install_different_branch_succeed, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index bd6ab0f9fa..38e354d3b9 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -76,6 +76,11 @@ def setUp(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) + from .subworkflows.info import ( + test_subworkflows_info_local, + test_subworkflows_info_remote, + test_subworkflows_info_remote_gitlab, + ) from .subworkflows.install import ( test_subworkflow_install_nopipeline, test_subworkflows_install_bam_sort_stats_samtools, From 8670fa7dfc505f994cdcda5a1d0c3e1cde86f9ac Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 13:39:35 +0100 Subject: [PATCH 562/854] fix prompt, add test from modules repo --- nf_core/__main__.py | 2 +- nf_core/components/info.py | 4 ++-- tests/modules/info.py | 14 ++++++++++++++ tests/subworkflows/info.py | 15 +++++++++++++++ tests/test_modules.py | 1 + tests/test_subworkflows.py | 1 + 6 files changed, 34 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ec73bee109..aaff680a7d 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -802,7 +802,7 @@ def info(ctx, tool, dir): ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - stdout.print(module_info.get_module_info()) + stdout.print(module_info.get_component_info()) except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/components/info.py b/nf_core/components/info.py index cfa71cc842..0417b274ce 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -117,8 +117,8 @@ def init_mod_name(self, component): ) else: components = self.modules_repo.get_avail_components(self.component_type) - module = questionary.autocomplete( - "Please select a {self.component_type[:-1]}", + component = questionary.autocomplete( + f"Please select a {self.component_type[:-1]}", choices=components, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() diff --git a/tests/modules/info.py b/tests/modules/info.py index 58d90a6afc..6c5b1063f1 100644 --- a/tests/modules/info.py +++ b/tests/modules/info.py @@ -47,3 +47,17 @@ def test_modules_info_local(self): assert "Module: trimgalore" in output assert "Inputs" in output assert "Outputs" in output + + +def test_modules_info_in_modules_repo(self): + """Test getting info about a module in the modules repo""" + mods_info = nf_core.modules.ModuleInfo(self.nfcore_modules, "fastqc") + mods_info.local = True + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Module: fastqc" in output + assert "Inputs" in output + assert "Outputs" in output diff --git a/tests/subworkflows/info.py b/tests/subworkflows/info.py index cda3d96f92..688120ac02 100644 --- a/tests/subworkflows/info.py +++ b/tests/subworkflows/info.py @@ -47,3 +47,18 @@ def test_subworkflows_info_local(self): assert "Subworkflow: bam_sort_stats_samtools" in output assert "Inputs" in output assert "Outputs" in output + + +def test_subworkflows_info_in_modules_repo(self): + """Test getting info about a locally subworkflow in the modules repo""" + self.subworkflow_install.install("bam_sort_stats_samtools") + mods_info = nf_core.subworkflows.SubworkflowInfo(self.nfcore_modules, "bam_sort_stats_samtools") + mods_info.local = True + mods_info_output = mods_info.get_component_info() + console = Console(record=True) + console.print(mods_info_output) + output = console.export_text() + + assert "Subworkflow: bam_sort_stats_samtools" in output + assert "Inputs" in output + assert "Outputs" in output diff --git a/tests/test_modules.py b/tests/test_modules.py index 9c3e1bf2c2..847c823799 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -136,6 +136,7 @@ def test_modulesrepo_class(self): test_modules_test_file_dict, ) from .modules.info import ( + test_modules_info_in_modules_repo, test_modules_info_local, test_modules_info_remote, test_modules_info_remote_gitlab, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 38e354d3b9..badf671803 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -77,6 +77,7 @@ def setUp(self): test_subworkflows_create_succeed, ) from .subworkflows.info import ( + test_subworkflows_info_in_modules_repo, test_subworkflows_info_local, test_subworkflows_info_remote, test_subworkflows_info_remote_gitlab, From 6516aa62d51a3c83028a7971aa1856d19edad762 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 17 Nov 2022 14:05:53 +0100 Subject: [PATCH 563/854] run isort --- nf_core/components/install.py | 3 ++- nf_core/components/update.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 592299aa90..a6d59135fe 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -3,9 +3,10 @@ import re from pathlib import Path +import questionary + import nf_core.modules.modules_utils import nf_core.utils -import questionary from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_json import ModulesJson diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 38d0f306f1..ac9bb5d936 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -4,9 +4,10 @@ import tempfile from pathlib import Path +import questionary + import nf_core.modules.modules_utils import nf_core.utils -import questionary from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_differ import ModulesDiffer From f520f6c1566768d008f153051dc0336ff6cdd3a9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 14:11:11 +0100 Subject: [PATCH 564/854] fix tests --- tests/test_subworkflows.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 6749d9f909..e1cd5d8355 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -2,6 +2,7 @@ """ import os +import shutil import tempfile import unittest @@ -68,12 +69,15 @@ def setUp(self): ) self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) - # Set up the nf-core/modules repo dummy - self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) - # Set up remove objects self.subworkflow_remove = nf_core.subworkflows.SubworkflowRemove(self.pipeline_dir) + def tearDown(self): + """Clean up temporary files and folders""" + + if os.path.exists(self.tmp_dir): + shutil.rmtree(self.tmp_dir) + ################################################ # Test of the individual subworkflow commands. # ################################################ From fe1ca2b58b18ddb85656cadacc0890de764b7e89 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 17 Nov 2022 14:14:52 +0100 Subject: [PATCH 565/854] exclusiely run prettier from pre-commit hook --- nf_core/lint_utils.py | 41 +--------------------------------------- tests/test_lint_utils.py | 33 -------------------------------- 2 files changed, 1 insertion(+), 73 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 5f59dad8e9..b86e1a853d 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,5 +1,4 @@ import logging -import shutil import subprocess from pathlib import Path @@ -55,45 +54,7 @@ def print_fixes(lint_obj): def run_prettier_on_file(file): - """Run Prettier on a file. - - Args: - file (Path | str): A file identifier as a string or pathlib.Path. - - Warns: - If Prettier is not installed, a warning is logged. - """ - - if shutil.which("prettier"): - _run_prettier_on_file(file) - elif shutil.which("pre-commit"): - _run_pre_commit_prettier_on_file(file) - else: - log.warning( - "Neither Prettier nor the prettier pre-commit hook are available. At least one of them is required." - ) - - -def _run_prettier_on_file(file): - """Run natively installed Prettier on a file.""" - - try: - subprocess.run( - ["prettier", "--write", file], - capture_output=True, - check=True, - ) - except subprocess.CalledProcessError as e: - if ": SyntaxError: " in e.stderr.decode(): - raise ValueError(f"Can't format {file} because it has a synthax error.\n{e.stderr.decode()}") from e - raise ValueError( - "There was an error running the prettier pre-commit hook.\n" - f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" - ) from e - - -def _run_pre_commit_prettier_on_file(file): - """Runs pre-commit hook prettier on a file if pre-commit is installed. + """Run the pre-commit hook prettier on a file. Args: file (Path | str): A file identifier as a string or pathlib.Path. diff --git a/tests/test_lint_utils.py b/tests/test_lint_utils.py index 88d5553346..d5af915fae 100644 --- a/tests/test_lint_utils.py +++ b/tests/test_lint_utils.py @@ -49,50 +49,17 @@ def git_dir_with_json_syntax_error(temp_git_repo): return file -@pytest.fixture -def ensure_prettier_is_not_found(monkeypatch): - def dont_find_prettier(x): - if x == "pre-commit": - which_x = WHICH_PRE_COMMIT - elif x == "prettier": - which_x = None - else: - raise ValueError(f"This mock is only inteded to hide prettier form tests. {x}") - return which_x - - monkeypatch.setattr("nf_core.lint_utils.shutil.which", dont_find_prettier) - - -@pytest.mark.skipif(shutil.which("prettier") is None, reason="Can't test prettier if it is not available.") def test_run_prettier_on_formatted_file(formatted_json): nf_core.lint_utils.run_prettier_on_file(formatted_json) assert formatted_json.read_text() == JSON_FORMATTED -def test_run_pre_commit_prettier_on_formatted_file(formatted_json, ensure_prettier_is_not_found): - nf_core.lint_utils.run_prettier_on_file(formatted_json) - assert formatted_json.read_text() == JSON_FORMATTED - - -@pytest.mark.skipif(shutil.which("prettier") is None, reason="Can't test prettier if it is not available.") def test_run_prettier_on_malformed_file(malformed_json): nf_core.lint_utils.run_prettier_on_file(malformed_json) assert malformed_json.read_text() == JSON_FORMATTED -def test_run_prettier_pre_commit_on_malformed_file(malformed_json, ensure_prettier_is_not_found): - nf_core.lint_utils.run_prettier_on_file(malformed_json) - assert malformed_json.read_text() == JSON_FORMATTED - - -@pytest.mark.skipif(shutil.which("prettier") is None, reason="Can't test prettier if it is not available.") def test_run_prettier_on_synthax_error_file(synthax_error_json): with pytest.raises(ValueError) as exc_info: nf_core.lint_utils.run_prettier_on_file(synthax_error_json) assert exc_info.value.args[0].startswith(f"Can't format {synthax_error_json} because it has a synthax error.") - - -def test_run_prettier_pre_commit_on_synthax_error_file(synthax_error_json, ensure_prettier_is_not_found): - with pytest.raises(ValueError) as exc_info: - nf_core.lint_utils.run_prettier_on_file(synthax_error_json) - assert exc_info.value.args[0].startswith(f"Can't format {synthax_error_json} because it has a synthax error.") From ecc0ea53a9a083541ad549f4f1465c0474f51aa6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 14:29:58 +0100 Subject: [PATCH 566/854] fix subworkflow info command --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index aaff680a7d..da19594e58 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1052,7 +1052,7 @@ def info(ctx, tool, dir): ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - stdout.print(subworkflow_info.get_subworkflow_info()) + stdout.print(subworkflow_info.get_component_info()) except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) From 5e305f7c246db7ac91258acf56378ba5d77f8aea Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 15:56:14 +0100 Subject: [PATCH 567/854] syntax highlighting for include hints --- nf_core/components/install.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index a6d59135fe..9acb37f98c 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -4,6 +4,8 @@ from pathlib import Path import questionary +from rich.console import Console +from rich.syntax import Syntax import nf_core.modules.modules_utils import nf_core.utils @@ -117,14 +119,22 @@ def install(self, component, silent=False): if not silent: # Print include statement component_name = "_".join(component.upper().split("/")) - log.info( - f"Include statement: include {{ {component_name} }} from '.{os.path.join(install_folder, component)}/main'" + log.info(f"Use the following statement to include this {self.component_type[:-1]}:") + Console().print( + Syntax( + f"include {{ {component_name} }} from '.{os.path.join(install_folder, component)}/main'", + "groovy", + theme="ansi_dark", + padding=1, + ) ) if self.component_type == "subworkflows": subworkflow_config = os.path.join(install_folder, component, "nextflow.config") if os.path.isfile(subworkflow_config): - log.info(f"Subworkflow config include statement: includeConfig '{subworkflow_config}'") - + log.info(f"Add the following config statement to use this subworkflow:") + Console().print( + Syntax(f"includeConfig '{subworkflow_config}'", "groovy", theme="ansi_dark", padding=1) + ) return True def get_modules_subworkflows_to_install(self, subworkflow_dir): From b0f7d590e48d61ef2ae8e1304ec065c9630163e7 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 16:07:00 +0100 Subject: [PATCH 568/854] update minimum nextflow version to 22.10.1 --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- nf_core/lint/readme.py | 4 ++-- .../pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- nf_core/pipeline-template/README.md | 4 ++-- nf_core/pipeline-template/nextflow.config | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 72b71ddd68..6cf8ffd921 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -33,7 +33,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 21.10.3)_ + * Nextflow version _(eg. 22.10.1)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 4e19bc65d2..be08f16154 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "21.10.3" + - "22.10.1" - "latest-everything" steps: # Get the repo code diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index b4b12621ce..b6c3166bd6 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "21.10.3" + - "22.10.1" - "latest-everything" steps: - uses: actions/checkout@v3 diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index a85a0979d3..daf92f7932 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -40,7 +40,7 @@ def readme(self): if "nextflow_badge" not in ignore_configs: # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) # and that it has the correct version nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" match = re.search(nf_badge_re, content) @@ -63,7 +63,7 @@ def readme(self): warned.append("README did not have a Nextflow minimum version badge.") # Check that the minimum version mentioned in the quick start section is consistent - # Looking for: "1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.10.3`)" + # Looking for: "1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`)" nf_version_re = r"1\.\s*Install\s*\[`Nextflow`\]\(https://www.nextflow.io/docs/latest/getstarted.html#installation\)\s*\(`>=(\d*\.\d*\.\d*)`\)" match = re.search(nf_version_re, content) if match: diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index 209c6e2d9f..27866452f0 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -42,7 +42,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 21.10.3)_ + * Nextflow version _(eg. 22.10.1)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_ diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 5e697a13cd..bf3dc36bc5 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "21.10.3" + - "22.10.1" - "latest-everything" steps: - name: Check out pipeline code diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 02a32f1f6e..0845f6aca0 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -10,7 +10,7 @@ {%- if github_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) @@ -46,7 +46,7 @@ The results obtained from the full-sized test can be viewed on the [nf-core webs ## Quick Start -1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=21.10.3`) +1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`) 2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_. diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 99acd8a0af..b80b28083d 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -194,7 +194,7 @@ manifest { homePage = 'https://github.com/{{ name }}' description = '{{ description }}' mainScript = 'main.nf' - nextflowVersion = '!>=21.10.3' + nextflowVersion = '!>=22.10.1' version = '{{ version }}' doi = '' } From 99b77c2530daea11d4abd5a3a6a89b435946d98e Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 17 Nov 2022 16:59:07 +0100 Subject: [PATCH 569/854] always print orphan include statements, print also if only modules.json was updated --- nf_core/components/remove.py | 66 ++++++++++++++++++++++----------- nf_core/lint/orphan_include.py | 0 nf_core/modules/modules_json.py | 16 +++++++- 3 files changed, 58 insertions(+), 24 deletions(-) delete mode 100644 nf_core/lint/orphan_include.py diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 4820b928ee..d14a391bf4 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -87,29 +87,51 @@ def remove(self, component): ) removed_component_dir = Path(component_type, repo_path, component_name) if removed_component: - if self.component_type == "subworkflows" and component_name != component: - # check if one of the dependent module/subworkflow has been manually included in the pipeline - include_stmts = self.check_if_in_include_stmts(str(removed_component_dir)) - if include_stmts: - # print the include statements - log.warn( - f"The {component_type[:-1]} '{component_name}' is still included in the following workflow file{nf_core.utils.plural_s(include_stmts)}:" - ) - console = Console() - for file, stmts in include_stmts.items(): - console.print(Rule(f"{file}", style="white")) - for stmt in stmts: - console.print( - Syntax( - stmt["line"], - "groovy", - theme="ansi_dark", - line_numbers=True, - start_line=stmt["line_number"], - padding=(0, 0, 1, 1), - ) + # check if one of the dependent module/subworkflow has been manually included in the pipeline + include_stmts = self.check_if_in_include_stmts(str(removed_component_dir)) + if include_stmts: + # print the include statements + log.warn( + f"The {component_type[:-1]} '{component_name}' is still included in the following workflow file{nf_core.utils.plural_s(include_stmts)}:" + ) + console = Console() + for file, stmts in include_stmts.items(): + console.print(Rule(f"{file}", style="white")) + for stmt in stmts: + console.print( + Syntax( + stmt["line"], + "groovy", + theme="ansi_dark", + line_numbers=True, + start_line=stmt["line_number"], + padding=(0, 0, 1, 1), ) - + ) + # ask the user if they still want to remove the component, install it otherwise + if not questionary.confirm( + f"Do you still want to remove the {component_type[:-1]} '{component_name}'?", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask(): + + current_version = modules_json.get_component_version( + self.component_type, + component_name, + self.modules_repo.remote_url, + self.modules_repo.repo_path, + ) + # install the component + if not modules_json.update( + self.component_type, + self.modules_repo, + component_name, + current_version, + self.component_type, + ): + log.warn( + f"Could not install the {component_type[:-1]} '{component_name}', please install it manually with 'nf-core {component_type} install {component_name}'." + ) + continue # Remove the component files of all entries removed from modules.json removed_components.append(component_name.replace("/", "_")) removed = ( diff --git a/nf_core/lint/orphan_include.py b/nf_core/lint/orphan_include.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 8064dc1564..db967bab1c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -665,15 +665,27 @@ def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=N repo_entry = self.modules_json["repos"][repo_url] if name in repo_entry[component_type].get(install_dir, {}): if removed_by in repo_entry[component_type][install_dir][name]["installed_by"]: - repo_entry[component_type][install_dir][name]["installed_by"].remove(removed_by) + self.modules_json["repos"][repo_url][component_type][install_dir][name]["installed_by"].remove( + removed_by + ) + # clean up empty entries if len(repo_entry[component_type][install_dir][name]["installed_by"]) == 0: self.modules_json["repos"][repo_url][component_type][install_dir].pop(name) if len(repo_entry[component_type][install_dir]) == 0: self.modules_json["repos"][repo_url].pop(component_type) + # write the updated modules.json file self.dump() return True + # write the updated modules.json file + if removed_by == component_type: + log.info( + f"Updated the 'installed_by' list of {name}, but it is still installed, because it is required by {repo_entry[component_type][install_dir][name]['installed_by']}." + ) + else: + log.info( + f"Removed {removed_by} from the 'installed_by' list of {name}, but it was also installed by other modules/subworkflows." + ) self.dump() - # Don't remove component entry from modules.json return False else: log.warning( From cfb8bb28ef1e10ef9260223b991863ac54fab93e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 18 Nov 2022 09:35:46 +0100 Subject: [PATCH 570/854] check modules structure only when installing modules --- nf_core/components/install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index a6d59135fe..64ac61419a 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -45,8 +45,9 @@ def install(self, component, silent=False): if not self.has_valid_directory(): return False - # Check modules directory structure - self.check_modules_structure() + if self.component_type == "modules": + # Check modules directory structure + self.check_modules_structure() # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) From f85856d204dae3d4f3d3d1520b6fa11d614f8bdf Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 18 Nov 2022 09:44:49 +0100 Subject: [PATCH 571/854] modify check_component_installed to make it more unerstandable --- nf_core/components/install.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 64ac61419a..4227fb295e 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -3,10 +3,9 @@ import re from pathlib import Path -import questionary - import nf_core.modules.modules_utils import nf_core.utils +import questionary from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_json import ModulesJson @@ -77,7 +76,7 @@ def install(self, component, silent=False): component_installed = self.check_component_installed( component, current_version, component_dir, self.modules_repo, self.force, self.prompt, silent ) - if not component_installed: + if component_installed: log.debug( f"{self.component_type[:-1].title()} is already installed and force is not set.\nAdding the new installation source {self.installed_by} for {self.component_type[:-1]} {component} to 'modules.json' without installing the {self.component_type}." ) @@ -197,7 +196,11 @@ def collect_and_verify_name(self, component, modules_repo): def check_component_installed(self, component, current_version, component_dir, modules_repo, force, prompt, silent): """ - Check that the module/subworkflow is not already installed + Check that the module/subworkflow is not already installed. + + Return: + False: if the component is not installed + True: if the component is installed """ if (current_version is not None and os.path.exists(component_dir)) and not force: if not silent: @@ -223,9 +226,9 @@ def check_component_installed(self, component, current_version, component_dir, m log.info( f"To update '{component}' run 'nf-core {self.component_type} {repo_flag}{branch_flag}update {component}'. To force reinstallation use '--force'" ) - return False + return True - return True + return False def get_version(self, component, sha, prompt, current_version, modules_repo): """ From 0c253348ad1649980a2282e3c9d1696b26df5fdc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 18 Nov 2022 09:48:12 +0100 Subject: [PATCH 572/854] run isort AGAIN --- nf_core/components/install.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 3a716273e6..99b2687ccc 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -3,15 +3,16 @@ import re from pathlib import Path +import questionary +from rich.console import Console +from rich.syntax import Syntax + import nf_core.modules.modules_utils import nf_core.utils -import questionary from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME -from rich.console import Console -from rich.syntax import Syntax log = logging.getLogger(__name__) From f171119ca4998862502899241e961a7af63bbd64 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 18 Nov 2022 10:31:24 +0100 Subject: [PATCH 573/854] revert changes because tests were failing --- nf_core/components/install.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 9acb37f98c..098de58e2f 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -3,16 +3,15 @@ import re from pathlib import Path -import questionary -from rich.console import Console -from rich.syntax import Syntax - import nf_core.modules.modules_utils import nf_core.utils +import questionary from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME +from rich.console import Console +from rich.syntax import Syntax log = logging.getLogger(__name__) @@ -75,10 +74,10 @@ def install(self, component, silent=False): component_dir = os.path.join(install_folder, component) # Check that the component is not already installed - component_installed = self.check_component_installed( + component_not_installed = self.check_component_installed( component, current_version, component_dir, self.modules_repo, self.force, self.prompt, silent ) - if not component_installed: + if not component_not_installed: log.debug( f"{self.component_type[:-1].title()} is already installed and force is not set.\nAdding the new installation source {self.installed_by} for {self.component_type[:-1]} {component} to 'modules.json' without installing the {self.component_type}." ) @@ -92,7 +91,7 @@ def install(self, component, silent=False): # Remove component if force is set and component is installed install_track = None - if self.force and component_installed: + if self.force: log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{component}'") self.clear_component_dir(component, component_dir) install_track = self.clean_modules_json(component, self.modules_repo, modules_json) @@ -131,7 +130,7 @@ def install(self, component, silent=False): if self.component_type == "subworkflows": subworkflow_config = os.path.join(install_folder, component, "nextflow.config") if os.path.isfile(subworkflow_config): - log.info(f"Add the following config statement to use this subworkflow:") + log.info("Add the following config statement to use this subworkflow:") Console().print( Syntax(f"includeConfig '{subworkflow_config}'", "groovy", theme="ansi_dark", padding=1) ) From 49dd5fc140de3816299ad57af85041b047657530 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 18 Nov 2022 10:49:41 +0100 Subject: [PATCH 574/854] this was modifyed by the merge :( --- nf_core/components/install.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index ed8b3790d3..2bcead44a7 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -209,8 +209,8 @@ def check_component_installed(self, component, current_version, component_dir, m Check that the module/subworkflow is not already installed. Return: - False: if the component is not installed - True: if the component is installed + True: if the component is not installed + False: if the component is installed """ if (current_version is not None and os.path.exists(component_dir)) and not force: if not silent: @@ -236,9 +236,9 @@ def check_component_installed(self, component, current_version, component_dir, m log.info( f"To update '{component}' run 'nf-core {self.component_type} {repo_flag}{branch_flag}update {component}'. To force reinstallation use '--force'" ) - return True + return False - return False + return True def get_version(self, component, sha, prompt, current_version, modules_repo): """ From 8581807b568641c62612b9a0cb1a55049045a2c9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 18 Nov 2022 11:04:15 +0100 Subject: [PATCH 575/854] run isort --- nf_core/components/install.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 2bcead44a7..4f3978be61 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -3,15 +3,16 @@ import re from pathlib import Path +import questionary +from rich.console import Console +from rich.syntax import Syntax + import nf_core.modules.modules_utils import nf_core.utils -import questionary from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import prompt_component_version_sha from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME -from rich.console import Console -from rich.syntax import Syntax log = logging.getLogger(__name__) From 2f2ddb5fd8387312fa0e206c4aa9b7d43901fda8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 18 Nov 2022 11:36:55 +0100 Subject: [PATCH 576/854] nicer print outputs --- nf_core/components/remove.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index d14a391bf4..d93e6f25de 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -2,8 +2,8 @@ from pathlib import Path import questionary -from rich.console import Console -from rich.rule import Rule +from rich.console import Console, Group +from rich.panel import Panel from rich.syntax import Syntax import nf_core.utils @@ -96,18 +96,26 @@ def remove(self, component): ) console = Console() for file, stmts in include_stmts.items(): - console.print(Rule(f"{file}", style="white")) + renderables = [] for stmt in stmts: - console.print( + renderables.append( Syntax( stmt["line"], "groovy", theme="ansi_dark", line_numbers=True, start_line=stmt["line_number"], - padding=(0, 0, 1, 1), ) ) + console.print( + Panel( + Group(*renderables), + title=f"{file}", + style="white", + title_align="center", + padding=1, + ) + ) # ask the user if they still want to remove the component, install it otherwise if not questionary.confirm( f"Do you still want to remove the {component_type[:-1]} '{component_name}'?", @@ -133,12 +141,14 @@ def remove(self, component): ) continue # Remove the component files of all entries removed from modules.json - removed_components.append(component_name.replace("/", "_")) removed = ( True if self.clear_component_dir(component, Path(self.dir, removed_component_dir)) or removed else False ) + # remember removed dependencies + if component_name != component: + removed_components.append(component_name.replace("/", "_")) if removed_components: - log.info(f"Removed files for {', '.join(removed_components)}") + log.info(f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'") return removed From b49342a94aa304cce5adc6e03f9dce8ae163ca32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 18 Nov 2022 11:44:34 +0100 Subject: [PATCH 577/854] Update nf_core/components/remove.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/remove.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index d14a391bf4..0ca7ccc01c 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -39,12 +39,11 @@ def remove(self, component): self.has_valid_directory() self.has_modules_file() - repo_dir = self.modules_repo.fullname repo_path = self.modules_repo.repo_path if component is None: component = questionary.autocomplete( f"{self.component_type[:-1]} name:", - choices=self.components_from_repo(repo_dir), + choices=self.components_from_repo(repo_path), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() From 81dc4e4e58e1a62df0d0e78d5f4a8a9b19a91e99 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 18 Nov 2022 11:47:55 +0100 Subject: [PATCH 578/854] handle singular removals --- nf_core/components/remove.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index c63b726eca..1fcea45163 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -149,5 +149,7 @@ def remove(self, component): if component_name != component: removed_components.append(component_name.replace("/", "_")) if removed_components: - log.info(f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'") + log.info(f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'.") + else: + log.info(f"Removed files for '{component}'.") return removed From cda7ab75ad5defe9f8f27739bd9415f681fb0c53 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 18 Nov 2022 13:39:05 +0100 Subject: [PATCH 579/854] add force option to remove, only print removal message if something was removed --- nf_core/components/remove.py | 65 +++++++++++++++++---------------- nf_core/modules/modules_json.py | 4 ++ 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 1fcea45163..568051d385 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -17,13 +17,14 @@ class ComponentRemove(ComponentCommand): def __init__(self, component_type, pipeline_dir): super().__init__(component_type, pipeline_dir) - def remove(self, component): + def remove(self, component, force=False): """ Remove an already installed module/subworkflow This command only works for modules/subworkflows that are installed from 'nf-core/modules' Args: component (str): Name of the component to remove + force (bool): Force removal of component, even if there is still an include statement in a workflow file Returns: bool: True if any item has been removed, False if not @@ -77,6 +78,9 @@ def remove(self, component): removed = False removed_components = [] for component_name, component_type in dependent_components.items(): + current_version = modules_json.get_component_version( + component_type, component_name, self.modules_repo.remote_url, repo_path + ) removed_component = modules_json.remove_entry( component_type, component_name, @@ -115,41 +119,38 @@ def remove(self, component): padding=1, ) ) - # ask the user if they still want to remove the component, install it otherwise - if not questionary.confirm( - f"Do you still want to remove the {component_type[:-1]} '{component_name}'?", - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask(): - - current_version = modules_json.get_component_version( - self.component_type, - component_name, - self.modules_repo.remote_url, - self.modules_repo.repo_path, - ) - # install the component - if not modules_json.update( - self.component_type, - self.modules_repo, - component_name, - current_version, - self.component_type, - ): - log.warn( - f"Could not install the {component_type[:-1]} '{component_name}', please install it manually with 'nf-core {component_type} install {component_name}'." - ) - continue + # ask the user if they still want to remove the component, add it back otherwise + if not force: + if not questionary.confirm( + f"Do you still want to remove the {component_type[:-1]} '{component_name}'?", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask(): + # add the component back to modules.json + if not modules_json.update( + self.component_type, + self.modules_repo, + component_name, + current_version, + self.component_type, + ): + log.warn( + f"Could not install the {component_type[:-1]} '{component_name}', please install it manually with 'nf-core {component_type} install {component_name}'." + ) + continue # Remove the component files of all entries removed from modules.json removed = ( True if self.clear_component_dir(component, Path(self.dir, removed_component_dir)) or removed else False ) - # remember removed dependencies - if component_name != component: - removed_components.append(component_name.replace("/", "_")) - if removed_components: - log.info(f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'.") - else: - log.info(f"Removed files for '{component}'.") + if removed: + # remember removed dependencies + if component_name != component: + removed_components.append(component_name.replace("/", "_")) + if removed_components: + log.info( + f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'." + ) + else: + log.info(f"Removed files for '{component}'.") return removed diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index db967bab1c..5d4d494d8b 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -612,6 +612,9 @@ def update( component_version (str): git SHA for the new module/subworkflow entry installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json' write_file (bool): whether to write the updated modules.json to a file. + + Returns: + bool: True if the module/subworkflow was successfully added to the 'modules.json' file """ if installed_by_log is None: installed_by_log = [] @@ -642,6 +645,7 @@ def update( self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) if write_file: self.dump() + return True def remove_entry(self, component_type, name, repo_url, install_dir, removed_by=None): """ From aab2c50682efabcd770f83a6d8e821be14249f0b Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 18 Nov 2022 15:12:07 +0100 Subject: [PATCH 580/854] fix tests --- nf_core/components/remove.py | 21 +++++++++++---------- tests/modules/lint.py | 6 +++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 568051d385..a2584c303f 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -143,14 +143,15 @@ def remove(self, component, force=False): if self.clear_component_dir(component, Path(self.dir, removed_component_dir)) or removed else False ) - if removed: - # remember removed dependencies - if component_name != component: - removed_components.append(component_name.replace("/", "_")) - if removed_components: - log.info( - f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'." - ) - else: - log.info(f"Removed files for '{component}'.") + + if removed: + # remember removed dependencies + if component_name != component: + removed_components.append(component_name.replace("/", "_")) + if removed: + # remember removed dependencies + if removed_components: + log.info(f"Removed files for '{component}' and it's dependencies '{', '.join(removed_components)}'.") + else: + log.info(f"Removed files for '{component}'.") return removed diff --git a/tests/modules/lint.py b/tests/modules/lint.py index d063188073..183276672b 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -35,9 +35,9 @@ def test_modules_lint_trimgalore(self): def test_modules_lint_empty(self): """Test linting a pipeline with no modules installed""" - self.mods_remove.remove("fastqc") - self.mods_remove.remove("multiqc") - self.mods_remove.remove("custom/dumpsoftwareversions") + self.mods_remove.remove("fastqc", force=True) + self.mods_remove.remove("multiqc", force=True) + self.mods_remove.remove("custom/dumpsoftwareversions", force=True) with pytest.raises(LookupError): nf_core.modules.ModuleLint(dir=self.pipeline_dir) From 4860579d43494377172a77a414bdef50a265acab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 18 Nov 2022 16:14:31 +0100 Subject: [PATCH 581/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- CHANGELOG.md | 2 +- nf_core/pipeline-template/nextflow.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02408a30b3..7d7377d6c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Fix lint warnings for `samplesheet_check.nf` module -- Add profile for running `docker` with the Apple M1 chips. +- Add profile for running `docker` with the ARM chips (including Apple silicon). ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 822f1e6321..fa94b2a1dd 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -109,7 +109,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false } - docker_m1 { + docker_arm { docker.enabled = true singularity.enabled = false podman.enabled = false From b421ab406235a2746b54450d7aa3ec183c6ed877 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 18 Nov 2022 16:29:57 +0100 Subject: [PATCH 582/854] switch to full reinstall for adding back included components --- nf_core/components/remove.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index a2584c303f..2215d8e8cf 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -10,6 +10,8 @@ from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson +from .install import ComponentInstall + log = logging.getLogger(__name__) @@ -34,7 +36,8 @@ def remove(self, component, force=False): return False # Check modules directory structure - self.check_modules_structure() + if self.component_type == "modules": + self.check_modules_structure() # Check whether pipeline is valid and with a modules.json file self.has_valid_directory() @@ -78,9 +81,6 @@ def remove(self, component, force=False): removed = False removed_components = [] for component_name, component_type in dependent_components.items(): - current_version = modules_json.get_component_version( - component_type, component_name, self.modules_repo.remote_url, repo_path - ) removed_component = modules_json.remove_entry( component_type, component_name, @@ -126,13 +126,7 @@ def remove(self, component, force=False): style=nf_core.utils.nfcore_question_style, ).unsafe_ask(): # add the component back to modules.json - if not modules_json.update( - self.component_type, - self.modules_repo, - component_name, - current_version, - self.component_type, - ): + if not ComponentInstall(self.dir, component_type, force=True).install(component_name): log.warn( f"Could not install the {component_type[:-1]} '{component_name}', please install it manually with 'nf-core {component_type} install {component_name}'." ) From 437fb8df6cdc611f7cb465a6986346e0f6b5ac00 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 21 Nov 2022 10:29:26 +0100 Subject: [PATCH 583/854] remove unnecessary inheritance from object --- nf_core/create.py | 2 +- nf_core/download.py | 2 +- nf_core/launch.py | 2 +- nf_core/licences.py | 2 +- nf_core/list.py | 6 +++--- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/modules_repo.py | 2 +- nf_core/modules/nfcore_module.py | 2 +- nf_core/schema.py | 2 +- nf_core/sync.py | 2 +- nf_core/utils.py | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index ed23c8487c..c7cdfc2269 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -26,7 +26,7 @@ log = logging.getLogger(__name__) -class PipelineCreate(object): +class PipelineCreate: """Creates a nf-core pipeline a la carte from the nf-core best-practice template. Args: diff --git a/nf_core/download.py b/nf_core/download.py index 5e7f4edb2f..2f964b3afd 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -64,7 +64,7 @@ def get_renderables(self): yield self.make_tasks_table([task]) -class DownloadWorkflow(object): +class DownloadWorkflow: """Downloads a nf-core workflow from GitHub to the local file system. Can also download its Singularity container image if required. diff --git a/nf_core/launch.py b/nf_core/launch.py index 03fcb6bce8..d03e112c3e 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -21,7 +21,7 @@ log = logging.getLogger(__name__) -class Launch(object): +class Launch: """Class to hold config option to launch a pipeline""" def __init__( diff --git a/nf_core/licences.py b/nf_core/licences.py index 3b188861eb..d686a56178 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -16,7 +16,7 @@ log = logging.getLogger(__name__) -class WorkflowLicences(object): +class WorkflowLicences: """A nf-core workflow licenses collection. Tries to retrieve the license information from all dependencies diff --git a/nf_core/list.py b/nf_core/list.py index b4353050e6..53307ac9bd 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -73,7 +73,7 @@ def get_local_wf(workflow, revision=None): return local_wf.local_path -class Workflows(object): +class Workflows: """Workflow container class. Is used to collect local and remote nf-core pipelines. Pipelines @@ -272,7 +272,7 @@ def print_json(self): ) -class RemoteWorkflow(object): +class RemoteWorkflow: """A information container for a remote workflow. Args: @@ -308,7 +308,7 @@ def __init__(self, data): ) -class LocalWorkflow(object): +class LocalWorkflow: """Class to handle local workflows pulled by nextflow""" def __init__(self, name): diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index de50770ebb..e6786a1e7a 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -35,7 +35,7 @@ class ModuleLintException(Exception): pass -class LintResult(object): +class LintResult: """An object to hold the results of a lint test""" def __init__(self, mod, lint_test, message, file_path): diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 673db73155..75ce795f5e 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -59,7 +59,7 @@ def update(self, op_code, cur_count, max_count=None, message=""): ) -class ModulesRepo(object): +class ModulesRepo: """ An object to store details about the repository being used for modules. diff --git a/nf_core/modules/nfcore_module.py b/nf_core/modules/nfcore_module.py index 67bfc6c1c4..e8bff1f686 100644 --- a/nf_core/modules/nfcore_module.py +++ b/nf_core/modules/nfcore_module.py @@ -4,7 +4,7 @@ from pathlib import Path -class NFCoreModule(object): +class NFCoreModule: """ A class to hold the information about a nf-core module Includes functionality for linting diff --git a/nf_core/schema.py b/nf_core/schema.py index 0be825fea2..d566f2b519 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -20,7 +20,7 @@ log = logging.getLogger(__name__) -class PipelineSchema(object): +class PipelineSchema: """Class to generate a schema object with functions to handle pipeline JSON Schema""" diff --git a/nf_core/sync.py b/nf_core/sync.py index 2384ad4de7..4d0e1e88ba 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -33,7 +33,7 @@ class PullRequestException(Exception): pass -class PipelineSync(object): +class PipelineSync: """Object to hold syncing information and results. Args: diff --git a/nf_core/utils.py b/nf_core/utils.py index 117c052e0a..a7ecb19a32 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -89,7 +89,7 @@ def rich_force_colors(): return None -class Pipeline(object): +class Pipeline: """Object to hold information about a local pipeline. Args: From c04d9090d94ad7bdf036b56bf4b4f21c1c28493b Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 21 Nov 2022 10:32:46 +0100 Subject: [PATCH 584/854] remove unused import --- nf_core/modules/modules_repo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 75ce795f5e..f9e8f31318 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -2,7 +2,6 @@ import logging import os import shutil -from importlib.resources import path from pathlib import Path import git From 17e80aa5f51c5c84a32c74c7d044f5461182303c Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 21 Nov 2022 12:32:19 +0100 Subject: [PATCH 585/854] code maintenance --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a424f1ec5..b50d0b76e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ - Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971)) - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out` - Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001)) -- Code maintenance ([#1818](https://github.com/nf-core/tools/pull/1818)) +- Code maintenance ([#1818](https://github.com/nf-core/tools/pull/1818), [#2032](https://github.com/nf-core/tools/pull/2032)) - Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999)) - Substitute ModulesCommand and SubworkflowsCommand by ComponentsCommand ([#2000](https://github.com/nf-core/tools/pull/2000)) - Don't print source file + line number on logging messages (except when verbose) ([#2015](https://github.com/nf-core/tools/pull/2015)) From 42d528bed0c38d7471417594a34f2cdf5f5bb36a Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 21 Nov 2022 14:27:04 +0100 Subject: [PATCH 586/854] Flip order of parmater summary and validation to prevent hiding of parameter errors --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 32425c20e9..e76780ceaf 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -63,16 +63,15 @@ class WorkflowMain { log.info "${workflow.manifest.name} ${workflow_version}" System.exit(0) } + + // Print parameter summary log to screen + log.info paramsSummaryLog(workflow, params, log) // Validate workflow parameters via the JSON schema if (params.validate_params) { NfcoreSchema.validateParameters(workflow, params, log) } - // Print parameter summary log to screen - - log.info paramsSummaryLog(workflow, params, log) - // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) From 3cd6b7477521269f62ad6d55fc6c3529fbe62c2c Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 21 Nov 2022 14:28:17 +0100 Subject: [PATCH 587/854] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a424f1ec5..d0b50234e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Check that the workflow name provided with a template doesn't contain dashes ([#1822](https://github.com/nf-core/tools/pull/1822)) - `nextflow run --version` will now print the workflow version from the manifest and exit ([#1951](https://github.com/nf-core/tools/pull/1951)) - Add profile for running `docker` with the ARM chips (including Apple silicon). +- Flip execution order of parameter summary printing and parameter validation to prevent 'hiding' of parameter errors ### Linting From cf1a57cb4ad8d30431698731fd6ad56c198f15a0 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 21 Nov 2022 14:31:50 +0100 Subject: [PATCH 588/854] Linting --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index e76780ceaf..714e659de9 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -63,7 +63,7 @@ class WorkflowMain { log.info "${workflow.manifest.name} ${workflow_version}" System.exit(0) } - + // Print parameter summary log to screen log.info paramsSummaryLog(workflow, params, log) From acd19ab4b480c3da272be85a4bcb009533ca293d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 21 Nov 2022 14:39:18 +0100 Subject: [PATCH 589/854] change docker_arm by arm --- nf_core/pipeline-template/nextflow.config | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 92e3c48396..77f6841fac 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -110,12 +110,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false } - docker_arm { - docker.enabled = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { From 9a9b8d3b63405d500515d8fe5ab295dba465dd60 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 21 Nov 2022 14:48:07 +0100 Subject: [PATCH 590/854] modify changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a424f1ec5..07000cce58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - Add codespaces template ([#1957](https://github.com/nf-core/tools/pull/1957)) - Check that the workflow name provided with a template doesn't contain dashes ([#1822](https://github.com/nf-core/tools/pull/1822)) - `nextflow run --version` will now print the workflow version from the manifest and exit ([#1951](https://github.com/nf-core/tools/pull/1951)) -- Add profile for running `docker` with the ARM chips (including Apple silicon). +- Add profile for running `docker` with the ARM chips (including Apple silicon) ([#1942](https://github.com/nf-core/tools/pull/1942) and [#2034](https://github.com/nf-core/tools/pull/2034)) ### Linting From 859ceab3f6297299afbf892fee6a94f1c2bcffa3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 21 Nov 2022 17:28:24 +0100 Subject: [PATCH 591/854] delete comment --- nf_core/create.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index ed23c8487c..a8b44c20cf 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -166,10 +166,16 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version + config_yml = nf_core.utils.load_tools_config() + if ( + "lint" in config_yml + and "nextflow_config" in config_yml["lint"] + and "manifest.name" in config_yml["lint"]["nextflow_config"] + ): + return param_dict, skip_paths # Check that the pipeline name matches the requirements if not re.match(r"^[a-z]+$", param_dict["short_name"]): - log.error("[red]Invalid workflow name: must be lowercase without punctuation.") - sys.exit(1) + raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") return param_dict, skip_paths From 7f3a9edefc9c618e60c6c5ab18486b338e4f04f5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 21 Nov 2022 17:31:04 +0100 Subject: [PATCH 592/854] modify changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07000cce58..aa20e20e06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### Linting +- Don't lint pipeline name if `manifest.name` in `.nf-core.yml` ([#2035](https://github.com/nf-core/tools/pull/2035)) + ### General - Refactor CLI flag `--hide-progress` to be at the top-level group, like `--verbose` ([#2016](https://github.com/nf-core/tools/pull/2016)) From 773b9948130ec4937eef3b38c7395976c5c0b892 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 21 Nov 2022 17:55:44 +0100 Subject: [PATCH 593/854] add docs for subworkflows --- README.md | 225 +++++++++++++++++++++++- docs/api/_src/api/subworkflows.md | 9 + nf_core/pipeline-template/docs/usage.md | 2 +- 3 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 docs/api/_src/api/subworkflows.md diff --git a/README.md b/README.md index 9b101ec9b3..49b229a205 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,17 @@ A python package with helper tools for the nf-core community. - [`modules bump-versions` - Bump software versions of modules](#bump-bioconda-and-container-versions-of-modules-in) - [`modules mulled` - Generate the name for a multi-tool container image](#generate-the-name-for-a-multi-tool-container-image) +- [`nf-core subworkflows` - commands for dealing with subworkflows](#subworkflows) + - [`subworkflows list` - List available subworkflows](#list-subworkflows) + - [`subworkflows list remote` - List remote subworkflows](#list-remote-subworkflows) + - [`subworkflows list local` - List installed subworkflows](#list-installed-subworkflows) + - [`subworkflows info` - Show information about a subworkflow](#show-information-about-a-subworkflow) + - [`subworkflows install` - Install subworkflows in a pipeline](#install-subworkflows-in-a-pipeline) + - [`subworkflows update` - Update subworkflows in a pipeline](#update-subworkflows-in-a-pipeline) + - [`subworkflows remove` - Remove a subworkflow from a pipeline](#remove-a-subworkflow-from-a-pipeline) + - [`subworkflows create` - Create a subworkflow from the template](#create-a-new-subworkflow) + - [`subworkflows create-test-yml` - Create the `test.yml` file for a subworkflow](#create-a-subworkflow-test-config-file) + - [`subworkflows test` - Run the tests for a subworkflow](#run-the-tests-for-a-subworkflow-using-pytest) - [Citation](#citation) The nf-core tools package is written in Python and can be imported and used within other packages. @@ -857,7 +868,7 @@ fake_command: nf-core modules create fastqc --author @nf-core-bot --label proce All modules on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. To help developers build new modules, the `nf-core modules create-test-yml` command automates the creation of the yaml file required to document the output file `md5sum` and other information generated by the testing. -After you have written a minimal Nextflow script to test your module `modules/tests/software///main.nf`, this command will run the tests for you and create the `modules/tests/software///test.yml` file. +After you have written a minimal Nextflow script to test your module `tests/modules///main.nf`, this command will run the tests for you and create the `tests/modules///test.yml` file. + +![`nf-core subworkflows list remote`](docs/images/nf-core-subworkflows-list-remote.svg) + +#### List installed subworkflows + +To list subworkflows installed in a local pipeline directory you can use `nf-core subworkflows list local`. This will list the subworkflows install in the current working directory by default. If you want to specify another directory, use the `--dir ` flag. + + + +![`nf-core subworkflows list local`](docs/images/nf-core-subworkflows-list-local.svg) + +## Show information about a subworkflow + +For quick help about how a subworkflow works, use `nf-core subworkflows info `. +This shows documentation about the subworkflow on the command line, similar to what's available on the +[nf-core website](https://nf-co.re/subworkflows). + + + +![`nf-core subworkflows info bam_rseqc`](docs/images/nf-core-subworkflows-info.svg) + +### Install subworkflows in a pipeline + +You can install subworkflows from [nf-core/modules](https://github.com/nf-core/modules) in your pipeline using `nf-core subworkflows install`. +A subworkflow installed this way will be installed to the `./subworkflows/nf-core/modules` directory. + + + +![`nf-core subworkflows install bam_rseqc`](docs/images/nf-core-subworkflows-install.svg) + +You can pass the subworkflow name as an optional argument to `nf-core subworkflows install` like above or select it from a list of available subworkflows by only running `nf-core subworkflows install`. + +There are four additional flags that you can use when installing a subworkflow: + +- `--dir`: Pipeline directory, the default is the current working directory. +- `--force`: Overwrite a previously installed version of the subworkflow. +- `--prompt`: Select the subworkflow version using a cli prompt. +- `--sha `: Install the subworkflow at a specific commit. + +### Update subworkflows in a pipeline + +You can update subworkflows installed from a remote repository in your pipeline using `nf-core subworkflows update`. + + + +![`nf-core subworkflows update --all --no-preview`](docs/images/nf-core-subworkflows-update.svg) + +You can pass the subworkflow name as an optional argument to `nf-core subworkflows update` like above or select it from the list of available subworkflows by only running `nf-core subworkflows update`. + +There are six additional flags that you can use with this command: + +- `--dir`: Pipeline directory, the default is the current working directory. +- `--force`: Reinstall subworkflow even if it appears to be up to date +- `--prompt`: Select the subworkflow version using a cli prompt. +- `--sha `: Install the subworkflow at a specific commit from the `nf-core/modules` repository. +- `--preview/--no-preview`: Show the diff between the installed files and the new version before installing. +- `--save-diff `: Save diffs to a file instead of updating in place. The diffs can then be applied with `git apply `. +- `--all`: Use this flag to run the command on all subworkflows in the pipeline. + +If you don't want to update certain subworkflows or want to update them to specific versions, you can make use of the `.nf-core.yml` configuration file. For example, you can prevent the `bam_rseqc` subworkflow installed from `nf-core/modules` from being updated by adding the following to the `.nf-core.yml` file: + +```yaml +update: + nf-core/modules: + bam_rseqc: False +``` + +If you want this subworkflow to be updated only to a specific version (or downgraded), you could instead specifiy the version: + +```yaml +update: + nf-core/modules: + bam_rseqc: "36a77f7c6decf2d1fb9f639ae982bc148d6828aa" +``` + +This also works at the repository level. For example, if you want to exclude all modules and subworkflows installed from `nf-core/modules` from being updated you could add: + +```yaml +update: + nf-core/modules: False +``` + +or if you want all subworkflows in `nf-core/modules` at a specific version: + +```yaml +update: + nf-core/modules: "e937c7950af70930d1f34bb961403d9d2aa81c7" +``` + +Note that the subworkflow versions specified in the `.nf-core.yml` file has higher precedence than versions specified with the command line flags, thus aiding you in writing reproducible pipelines. + +### Remove a subworkflow from a pipeline + +To delete a subworkflow from your pipeline, run `nf-core subworkflows remove`. + + + +![`nf-core subworkflows remove bam_rseqc`](docs/images/nf-core-subworkflows-remove.svg) + +You can pass the subworkflow name as an optional argument to `nf-core subworkflows remove` like above or select it from the list of available subworkflows by only running `nf-core subworkflows remove`. To specify the pipeline directory, use `--dir `. + +### Create a new subworkflow + +This command creates a new nf-core subworkflow from the nf-core subworkflow template. +This ensures that your subworkflow follows the nf-core guidelines. +The template contains extensive `TODO` messages to walk you through the changes you need to make to the template. + +You can create a new subworkflow using `nf-core subworkflows create`. + +This command can be used both when writing a subworkflow for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, +and also when creating local subworkflows for a pipeline. + +Which type of repository you are working in is detected by the `repository_type` flag in a `.nf-core.yml` file in the root directory, +set to either `pipeline` or `modules`. +The command will automatically look through parent directories for this file to set the root path, so that you can run the command in a subdirectory. +It will start in the current working directory, or whatever is specified with `--dir `. + +The `nf-core subworkflows create` command will prompt you with the relevant questions in order to create all of the necessary subworkflow files. + + + +![`cd modules && nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --label process_low --meta --force`](docs/images/nf-core-subworkflows-create.svg) + +### Create a subworkflow test config file + +All subworkflows on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. +To help developers build new subworkflows, the `nf-core subworkflows create-test-yml` command automates the creation of the yaml file required to document the output file `md5sum` and other information generated by the testing. +After you have written a minimal Nextflow script to test your subworkflow in `/tests/subworkflow//main.nf`, this command will run the tests for you and create the `/tests/subworkflow///test.yml` file. + + + +![`nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force`](docs/images/nf-core-subworkflows-create-test.svg) + +### Run the tests for a subworkflow using pytest + +To run unit tests of a subworkflow that you have installed or the test created by the command [`nf-core subworkflow create-test-yml`](#create-a-subworkflow-test-config-file), you can use `nf-core subworkflows test` command. This command runs the tests specified in `tests/subworkflows//test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/). + +You can specify the subworkflow name in the form TOOL/SUBTOOL in command line or provide it later by prompts. + + + +![`nf-core subworkflows test bam_rseqc --no-prompts`](docs/images/nf-core-subworkflows-test.svg) + ## Citation If you use `nf-core tools` in your work, please cite the `nf-core` publication as follows: diff --git a/docs/api/_src/api/subworkflows.md b/docs/api/_src/api/subworkflows.md new file mode 100644 index 0000000000..438ccd0185 --- /dev/null +++ b/docs/api/_src/api/subworkflows.md @@ -0,0 +1,9 @@ +# nf_core.subworkflows + +```{eval-rst} +.. automodule:: nf_core.subworkflows + :members: + :undoc-members: + :show-inheritance: + :private-members: +``` diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 843298a3d9..9a171f5aad 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -183,7 +183,7 @@ A first step to bypass this error, you could try to increase the amount of CPUs, #### Advanced option on process level To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). -We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/software/star/align/main.nf`. +We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/star/align/main.nf`. If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. From 7b5e4e77f0d9a84871f41da62046c1fc1e741451 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 21 Nov 2022 17:59:00 +0100 Subject: [PATCH 594/854] add link to step-by-step-guide --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 49b229a205..cf85a943e5 100644 --- a/README.md +++ b/README.md @@ -1101,6 +1101,7 @@ You can pass the subworkflow name as an optional argument to `nf-core subworkflo This command creates a new nf-core subworkflow from the nf-core subworkflow template. This ensures that your subworkflow follows the nf-core guidelines. The template contains extensive `TODO` messages to walk you through the changes you need to make to the template. +See the [subworkflow documentation](https://nf-co.re/docs/contributing/subworkflows) for more details around creating a new subworkflow, including rules about nomenclature and a step-by-step guide. You can create a new subworkflow using `nf-core subworkflows create`. From d376411fed96895fbe7e383ab637162d77652edd Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 22 Nov 2022 09:47:46 +0100 Subject: [PATCH 595/854] linting --- nf_core/components/update.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index ac9bb5d936..4a8420ccb9 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -146,6 +146,9 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr exit_value = True all_patches_successful = True for modules_repo, component, sha, patch_relpath in components_info: + if component is None: + # The entry from .nf-core.yml is set to false, skip update of this component + continue component_fullname = str(Path(self.component_type, modules_repo.repo_path, component)) # Are we updating the files in place or not? dry_run = self.show_diff or self.save_diff_fn @@ -386,12 +389,13 @@ def get_single_component_info(self, component): config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(component) if config_entry is not None and config_entry is not True: if config_entry is False: - raise UserWarning( - f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is set to False" + log.warn( + f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' for '{component}' is set to False" ) + return (self.modules_repo, None, None, None) if not isinstance(config_entry, str): raise UserWarning( - f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' is of wrong type" + f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' for '{component}' is of wrong type" ) sha = config_entry From bc7bed973691fd20c2ab0b44312431c132f2cfb0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 22 Nov 2022 09:54:13 +0100 Subject: [PATCH 596/854] update examples for ignore update settings --- README.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index cf85a943e5..4fd48f7140 100644 --- a/README.md +++ b/README.md @@ -781,30 +781,34 @@ If you don't want to update certain modules or want to update them to specific v ```yaml update: - nf-core/modules: - star/align: False + https://github.com/nf-core/modules.git: + nf-core: + star/align: False ``` If you want this module to be updated only to a specific version (or downgraded), you could instead specifiy the version: ```yaml update: - nf-core/modules: - star/align: "e937c7950af70930d1f34bb961403d9d2aa81c7" + https://github.com/nf-core/modules.git: + nf-core: + star/align: "e937c7950af70930d1f34bb961403d9d2aa81c7" ``` This also works at the repository level. For example, if you want to exclude all modules installed from `nf-core/modules` from being updated you could add: ```yaml update: - nf-core/modules: False + https://github.com/nf-core/modules.git: + nf-core: False ``` or if you want all modules in `nf-core/modules` at a specific version: ```yaml update: - nf-core/modules: "e937c7950af70930d1f34bb961403d9d2aa81c7" + https://github.com/nf-core/modules.git: + nf-core: "e937c7950af70930d1f34bb961403d9d2aa81c7" ``` Note that the module versions specified in the `.nf-core.yml` file has higher precedence than versions specified with the command line flags, thus aiding you in writing reproducible pipelines. @@ -1056,30 +1060,34 @@ If you don't want to update certain subworkflows or want to update them to speci ```yaml update: - nf-core/modules: - bam_rseqc: False + https://github.com/nf-core/modules.git: + nf-core: + bam_rseqc: False ``` If you want this subworkflow to be updated only to a specific version (or downgraded), you could instead specifiy the version: ```yaml update: - nf-core/modules: - bam_rseqc: "36a77f7c6decf2d1fb9f639ae982bc148d6828aa" + https://github.com/nf-core/modules.git: + nf-core: + bam_rseqc: "36a77f7c6decf2d1fb9f639ae982bc148d6828aa" ``` This also works at the repository level. For example, if you want to exclude all modules and subworkflows installed from `nf-core/modules` from being updated you could add: ```yaml update: - nf-core/modules: False + https://github.com/nf-core/modules.git: + nf-core: False ``` or if you want all subworkflows in `nf-core/modules` at a specific version: ```yaml update: - nf-core/modules: "e937c7950af70930d1f34bb961403d9d2aa81c7" + https://github.com/nf-core/modules.git: + nf-core: "e937c7950af70930d1f34bb961403d9d2aa81c7" ``` Note that the subworkflow versions specified in the `.nf-core.yml` file has higher precedence than versions specified with the command line flags, thus aiding you in writing reproducible pipelines. From b6b8555393af17ce919b4b077da30cd860c6da81 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 22 Nov 2022 10:10:04 +0100 Subject: [PATCH 597/854] linting --- nf_core/components/update.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 4a8420ccb9..3e2130e022 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -381,6 +381,19 @@ def get_single_component_info(self, component): sha = self.sha config_entry = None + if any( + [ + entry.count("/") == 1 + and (entry.endswith("modules") or entry.endswith("subworkflows")) + and not (entry.endswith(".git") or entry.endswith(".git/")) + for entry in self.update_config.keys() + ] + ): + raise UserWarning( + "Your '.nf-core.yml' file format is outdated. " + "The format should be of the form:\n" + "update:\n :\n :\n :" + ) if isinstance(self.update_config.get(self.modules_repo.remote_url, {}), str): # If the repo entry is a string, it's the sha to update to config_entry = self.update_config.get(self.modules_repo.remote_url, {}) From e67b89a68c9d80d656680c8cfd4f1e68fc167103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 22 Nov 2022 10:40:28 +0100 Subject: [PATCH 598/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4fd48f7140..0f21412215 100644 --- a/README.md +++ b/README.md @@ -1017,7 +1017,7 @@ working_dir: tmp/nf-core-nextbigthing ### Install subworkflows in a pipeline You can install subworkflows from [nf-core/modules](https://github.com/nf-core/modules) in your pipeline using `nf-core subworkflows install`. -A subworkflow installed this way will be installed to the `./subworkflows/nf-core/modules` directory. +A subworkflow installed this way will be installed to the `./subworkflows/nf-core` directory. - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing'create.py:236 -INFO     Initialising pipeline git repository                                          create.py:538 -INFO     Done. Remember to add a remote and push to GitHub:                            create.py:545 - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for create.py:551 -         syncing.                                                                       -INFO    !!!!!! IMPORTANT !!!!!!create.py:227 - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' +INFO     Initialising pipeline git repository                                                        +INFO     Done. Remember to add a remote and push to GitHub:                                          + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      +INFO    !!!!!! IMPORTANT !!!!!! + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 51be23194a..6b19cc6433 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,77 +19,76 @@ font-weight: 700; } - .terminal-1992655221-matrix { + .terminal-3721474111-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1992655221-title { + .terminal-3721474111-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1992655221-r1 { fill: #c5c8c6 } -.terminal-1992655221-r2 { fill: #98a84b } -.terminal-1992655221-r3 { fill: #9a9b99 } -.terminal-1992655221-r4 { fill: #608ab1 } -.terminal-1992655221-r5 { fill: #d0b344 } -.terminal-1992655221-r6 { fill: #868887 } + .terminal-3721474111-r1 { fill: #c5c8c6 } +.terminal-3721474111-r2 { fill: #98a84b } +.terminal-3721474111-r3 { fill: #9a9b99 } +.terminal-3721474111-r4 { fill: #608ab1 } +.terminal-3721474111-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +100,26 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq'download.py:159 -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                                      download.py:162 -INFO     Downloading centralised configs from GitHub                                 download.py:166 + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq' +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                                                      +INFO     Downloading centralised configs from GitHub                                                 diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index a52e1d7891..55a70279d9 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,73 +19,72 @@ font-weight: 700; } - .terminal-100502750-matrix { + .terminal-1175629223-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-100502750-title { + .terminal-1175629223-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-100502750-r1 { fill: #c5c8c6 } -.terminal-100502750-r2 { fill: #98a84b } -.terminal-100502750-r3 { fill: #9a9b99 } -.terminal-100502750-r4 { fill: #608ab1 } -.terminal-100502750-r5 { fill: #d0b344 } -.terminal-100502750-r6 { fill: #868887 } -.terminal-100502750-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-100502750-r8 { fill: #68a0b3;font-weight: bold } + .terminal-1175629223-r1 { fill: #c5c8c6 } +.terminal-1175629223-r2 { fill: #98a84b } +.terminal-1175629223-r3 { fill: #9a9b99 } +.terminal-1175629223-r4 { fill: #608ab1 } +.terminal-1175629223-r5 { fill: #d0b344 } +.terminal-1175629223-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1175629223-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,24 +96,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by        launch.py:131 -         Nextflow config files or profiles                                              - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1)list.py:67 + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      +         files or profiles                                                                           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 4173e2faa0..db45603a7b 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,109 +19,108 @@ font-weight: 700; } - .terminal-1928727807-matrix { + .terminal-2422327830-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1928727807-title { + .terminal-2422327830-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1928727807-r1 { fill: #c5c8c6 } -.terminal-1928727807-r2 { fill: #98a84b } -.terminal-1928727807-r3 { fill: #9a9b99 } -.terminal-1928727807-r4 { fill: #608ab1 } -.terminal-1928727807-r5 { fill: #d0b344 } -.terminal-1928727807-r6 { fill: #68a0b3;font-weight: bold } -.terminal-1928727807-r7 { fill: #868887 } -.terminal-1928727807-r8 { fill: #c5c8c6;font-weight: bold } + .terminal-2422327830-r1 { fill: #c5c8c6 } +.terminal-2422327830-r2 { fill: #98a84b } +.terminal-2422327830-r3 { fill: #9a9b99 } +.terminal-2422327830-r4 { fill: #608ab1 } +.terminal-2422327830-r5 { fill: #d0b344 } +.terminal-2422327830-r6 { fill: #68a0b3;font-weight: bold } +.terminal-2422327830-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +132,36 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                     licences.py:77 -INFO     Warning: This tool only prints licence information for the software tools    licences.py:98 -         packaged using conda.                                                         -INFO     The pipeline may use other software and dependencies not described here.     licences.py:99 -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    +INFO     Warning: This tool only prints licence information for the software tools packaged using    +         conda.                                                                                      +INFO     The pipeline may use other software and dependencies not described here.                    +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index d5b62b7c01..b255a7c4de 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Testing pipeline: .__init__.py:265 - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│samplesheet_check│modules/local/sampleshe…│Process label unspecified│ -│samplesheet_check│modules/local/sampleshe…│when: condition has been │ -│││removed│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 178 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   2 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/custom/…│New version available│ +│fastqc│modules/nf-core/fastqc  │New version available│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 178 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   2 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 00b4170ed2..0dfaae8459 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-1720318354-matrix { + .terminal-1440873036-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1720318354-title { + .terminal-1440873036-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1720318354-r1 { fill: #c5c8c6 } -.terminal-1720318354-r2 { fill: #98a84b } -.terminal-1720318354-r3 { fill: #9a9b99 } -.terminal-1720318354-r4 { fill: #608ab1 } -.terminal-1720318354-r5 { fill: #d0b344 } -.terminal-1720318354-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1720318354-r7 { fill: #868887 } + .terminal-1440873036-r1 { fill: #c5c8c6 } +.terminal-1440873036-r2 { fill: #98a84b } +.terminal-1440873036-r3 { fill: #9a9b99 } +.terminal-1440873036-r4 { fill: #608ab1 } +.terminal-1440873036-r5 { fill: #d0b344 } +.terminal-1440873036-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1440873036-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   510 │            3.9 │   4 days ago │           - │ -                   │ -│ rnafusion            │    80 │          2.1.0 │ 3 months ago │           - │ -                   │ -│ smrnaseq             │    44 │          2.0.0 │ 4 months ago │           - │ -                   │ -│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    20 │            dev │            - │           - │ -                   │ -│ lncpipe              │    22 │            dev │            - │           - │ -                   │ -│ scflow               │    13 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq             │    44 │          2.1.0 │ 1 months ago │           - │ -                   │ +│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ +│ rnafusion            │    83 │          2.1.0 │ 5 months ago │           - │ -                   │ +│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    24 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    14 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 5c61a13a3a..80d81c4dcd 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-1833040706-matrix { + .terminal-2329934618-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1833040706-title { + .terminal-2329934618-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1833040706-r1 { fill: #c5c8c6 } -.terminal-1833040706-r2 { fill: #98a84b } -.terminal-1833040706-r3 { fill: #9a9b99 } -.terminal-1833040706-r4 { fill: #608ab1 } -.terminal-1833040706-r5 { fill: #d0b344 } -.terminal-1833040706-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1833040706-r7 { fill: #868887 } -.terminal-1833040706-r8 { fill: #868887;font-style: italic; } + .terminal-2329934618-r1 { fill: #c5c8c6 } +.terminal-2329934618-r2 { fill: #98a84b } +.terminal-2329934618-r3 { fill: #9a9b99 } +.terminal-2329934618-r4 { fill: #608ab1 } +.terminal-2329934618-r5 { fill: #d0b344 } +.terminal-2329934618-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2329934618-r7 { fill: #868887 } +.terminal-2329934618-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   510 │            3.9 │   4 days ago │           - │ -                   │ -│ sarek                │   193 │          3.0.2 │  1 weeks ago │           - │ -                   │ -│ chipseq              │   127 │          2.0.0 │    yesterday │           - │ -                   │ -│ atacseq              │   117 │          1.2.2 │ 5 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ +│ sarek                │   210 │          3.1.1 │  2 weeks ago │           - │ -                   │ +│ chipseq              │   133 │          2.0.0 │ 2 months ago │           - │ -                   │ +│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 3c20bf2e71..efd90f47f8 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-602830802-matrix { + .terminal-2923789563-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-602830802-title { + .terminal-2923789563-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-602830802-r1 { fill: #c5c8c6 } -.terminal-602830802-r2 { fill: #98a84b } -.terminal-602830802-r3 { fill: #9a9b99 } -.terminal-602830802-r4 { fill: #608ab1 } -.terminal-602830802-r5 { fill: #d0b344 } -.terminal-602830802-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-602830802-r7 { fill: #868887 } -.terminal-602830802-r8 { fill: #868887;font-style: italic; } + .terminal-2923789563-r1 { fill: #c5c8c6 } +.terminal-2923789563-r2 { fill: #98a84b } +.terminal-2923789563-r3 { fill: #9a9b99 } +.terminal-2923789563-r4 { fill: #608ab1 } +.terminal-2923789563-r5 { fill: #d0b344 } +.terminal-2923789563-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2923789563-r7 { fill: #868887 } +.terminal-2923789563-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ chipseq              │   127 │          2.0.0 │    yesterday │           - │ -                   │ -│ rnaseq               │   510 │            3.9 │   4 days ago │           - │ -                   │ -│ isoseq               │     5 │          1.1.1 │   1 week ago │           - │ -                   │ -│ sarek                │   193 │          3.0.2 │  1 weeks ago │           - │ -                   │ -│ airrflow             │    19 │          2.3.0 │  3 weeks ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ ampliseq             │   101 │          2.4.1 │ 20 hours ago │           - │ -                   │ +│ airrflow             │    24 │          2.4.0 │   2 days ago │           - │ -                   │ +│ mhcquant             │    21 │          2.4.0 │   6 days ago │           - │ -                   │ +│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ +│ methylseq            │    95 │          2.2.0 │  1 weeks ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index ee5dd12b1b..9f760347a3 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-578811831-matrix { + .terminal-824833976-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-578811831-title { + .terminal-824833976-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-578811831-r1 { fill: #c5c8c6 } -.terminal-578811831-r2 { fill: #98a84b } -.terminal-578811831-r3 { fill: #9a9b99 } -.terminal-578811831-r4 { fill: #608ab1 } -.terminal-578811831-r5 { fill: #d0b344 } -.terminal-578811831-r6 { fill: #98a84b;font-weight: bold } -.terminal-578811831-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-824833976-r1 { fill: #c5c8c6 } +.terminal-824833976-r2 { fill: #98a84b } +.terminal-824833976-r3 { fill: #9a9b99 } +.terminal-824833976-r4 { fill: #608ab1 } +.terminal-824833976-r5 { fill: #d0b344 } +.terminal-824833976-r6 { fill: #98a84b;font-weight: bold } +.terminal-824833976-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 8f05428006..a9bf5835bc 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Looking for test workflow entry points:                             test_yml_builder.py:128 -'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc_single_end'test_yml_builder.py:162 -INFO     Running 'fastqc' test with command:                                 test_yml_builder.py:330 -nextflow run ./tests/modules/nf-core/fastqc -entry  -test_fastqc_single_end -c ./tests/config/nextflow.config -c  -./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpha4zaib5 -work-dir /tmp/tmp1swft0ph + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpk231usbb -work-dir /tmp/tmpvt2u8pj3 diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index b845061eb0..90626c9a55 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Repository type: modulescreate.py:99 -INFO    Press enter to use default values (shown in brackets)or type your own create.py:103 -responses. ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9'create.py:171 -INFO     Could not find a Docker/Singularity container (Unexpected response code `503` create.py:200 -         for                                                                            -https://api.biocontainers.pro/ga4gh/trs/v2/tools/fastqc/versions/fastqc-0.11. -9) + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9' +INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1' +INFO     Using Singularity container:                                                                +'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' +INFO     Created / edited following files:                                                           +           ./modules/nf-core/fastqc/main.nf +           ./modules/nf-core/fastqc/meta.yml +           ./tests/modules/nf-core/fastqc/main.nf +           ./tests/modules/nf-core/fastqc/test.yml +           ./tests/modules/nf-core/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 546a92b854..b22069ca8c 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-4091539959-matrix { + .terminal-377483768-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4091539959-title { + .terminal-377483768-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4091539959-r1 { fill: #c5c8c6 } -.terminal-4091539959-r2 { fill: #98a84b } -.terminal-4091539959-r3 { fill: #9a9b99 } -.terminal-4091539959-r4 { fill: #608ab1 } -.terminal-4091539959-r5 { fill: #d0b344 } -.terminal-4091539959-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4091539959-r7 { fill: #98a84b;font-weight: bold } -.terminal-4091539959-r8 { fill: #868887 } -.terminal-4091539959-r9 { fill: #d08442 } -.terminal-4091539959-r10 { fill: #868887;font-style: italic; } -.terminal-4091539959-r11 { fill: #98729f } + .terminal-377483768-r1 { fill: #c5c8c6 } +.terminal-377483768-r2 { fill: #98a84b } +.terminal-377483768-r3 { fill: #9a9b99 } +.terminal-377483768-r4 { fill: #608ab1 } +.terminal-377483768-r5 { fill: #d0b344 } +.terminal-377483768-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-377483768-r7 { fill: #98a84b;font-weight: bold } +.terminal-377483768-r8 { fill: #868887 } +.terminal-377483768-r9 { fill: #d08442 } +.terminal-377483768-r10 { fill: #868887;font-style: italic; } +.terminal-377483768-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index def3a19771..4a53bd06c8 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Installing 'abacas'install.py:131 -INFO     Include statement: include { ABACAS } from '../modules/nf-core/abacas/main'install.py:140 + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index fdaf8e8bad..0531cdfe79 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -19,95 +19,94 @@ font-weight: 700; } - .terminal-663186508-matrix { + .terminal-2882701330-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-663186508-title { + .terminal-2882701330-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-663186508-r1 { fill: #c5c8c6 } -.terminal-663186508-r2 { fill: #98a84b } -.terminal-663186508-r3 { fill: #9a9b99 } -.terminal-663186508-r4 { fill: #608ab1 } -.terminal-663186508-r5 { fill: #d0b344 } -.terminal-663186508-r6 { fill: #868887 } -.terminal-663186508-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-663186508-r8 { fill: #98a84b;font-weight: bold } -.terminal-663186508-r9 { fill: #cc555a } + .terminal-2882701330-r1 { fill: #c5c8c6 } +.terminal-2882701330-r2 { fill: #98a84b } +.terminal-2882701330-r3 { fill: #9a9b99 } +.terminal-2882701330-r4 { fill: #608ab1 } +.terminal-2882701330-r5 { fill: #d0b344 } +.terminal-2882701330-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2882701330-r7 { fill: #98a84b;font-weight: bold } +.terminal-2882701330-r8 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -119,31 +118,31 @@ - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Linting modules repo: '.'__init__.py:200 -INFO     Linting module: 'multiqc'__init__.py:204 - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  23 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' + +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  23 Tests Passed │ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 02bc53bd4c..0d5cdf6f2f 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,109 +19,108 @@ font-weight: 700; } - .terminal-3572137377-matrix { + .terminal-2639661206-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3572137377-title { + .terminal-2639661206-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3572137377-r1 { fill: #c5c8c6 } -.terminal-3572137377-r2 { fill: #98a84b } -.terminal-3572137377-r3 { fill: #9a9b99 } -.terminal-3572137377-r4 { fill: #608ab1 } -.terminal-3572137377-r5 { fill: #d0b344 } -.terminal-3572137377-r6 { fill: #868887 } -.terminal-3572137377-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-3572137377-r8 { fill: #868887;font-style: italic; } + .terminal-2639661206-r1 { fill: #c5c8c6 } +.terminal-2639661206-r2 { fill: #98a84b } +.terminal-2639661206-r3 { fill: #9a9b99 } +.terminal-2639661206-r4 { fill: #608ab1 } +.terminal-2639661206-r5 { fill: #d0b344 } +.terminal-2639661206-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2639661206-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +132,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                       list.py:138 - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 2d43fab033..0f37822110 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,110 +19,109 @@ font-weight: 700; } - .terminal-1649706811-matrix { + .terminal-3342753500-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1649706811-title { + .terminal-3342753500-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1649706811-r1 { fill: #c5c8c6 } -.terminal-1649706811-r2 { fill: #98a84b } -.terminal-1649706811-r3 { fill: #9a9b99 } -.terminal-1649706811-r4 { fill: #608ab1 } -.terminal-1649706811-r5 { fill: #d0b344 } -.terminal-1649706811-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-1649706811-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-1649706811-r8 { fill: #868887 } -.terminal-1649706811-r9 { fill: #868887;font-style: italic; } + .terminal-3342753500-r1 { fill: #c5c8c6 } +.terminal-3342753500-r2 { fill: #98a84b } +.terminal-3342753500-r3 { fill: #9a9b99 } +.terminal-3342753500-r4 { fill: #608ab1 } +.terminal-3342753500-r5 { fill: #d0b344 } +.terminal-3342753500-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-3342753500-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3342753500-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -134,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):         list.py:133 - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                   │ -│ abricate/run                             │ -│ abricate/summary                         │ -│ adapterremoval                           │ -│ adapterremovalfixprefix                  │ -│ agrvate                                  │ -│ allelecounter                            │ -│ ampir                                    │ -│ amplify/predict                          │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ agat/convertspgff2gtf                                 │ +│ agat/convertspgxf2gxf                                 │ +│ agat/spstatistics                                     │ +│ agat/sqstatbasic                                      │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index 3581be2c97..5373182936 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -19,70 +19,69 @@ font-weight: 700; } - .terminal-670140257-matrix { + .terminal-3390485959-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-670140257-title { + .terminal-3390485959-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-670140257-r1 { fill: #c5c8c6 } -.terminal-670140257-r2 { fill: #98a84b } -.terminal-670140257-r3 { fill: #9a9b99 } -.terminal-670140257-r4 { fill: #608ab1 } -.terminal-670140257-r5 { fill: #d0b344 } -.terminal-670140257-r6 { fill: #868887 } -.terminal-670140257-r7 { fill: #00823d;font-weight: bold } -.terminal-670140257-r8 { fill: #68a0b3;font-weight: bold } + .terminal-3390485959-r1 { fill: #c5c8c6 } +.terminal-3390485959-r2 { fill: #98a84b } +.terminal-3390485959-r3 { fill: #9a9b99 } +.terminal-3390485959-r4 { fill: #608ab1 } +.terminal-3390485959-r5 { fill: #d0b344 } +.terminal-3390485959-r6 { fill: #00823d;font-weight: bold } +.terminal-3390485959-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -94,23 +93,23 @@ - + - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                              mulled.py:68 -INFO     Mulled container hash:                                                      __main__.py:834 -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                                           +INFO     Mulled container hash:                                                                      +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 77432e0e75..3bc3f80ffc 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,66 +19,65 @@ font-weight: 700; } - .terminal-1687321738-matrix { + .terminal-239277571-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1687321738-title { + .terminal-239277571-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1687321738-r1 { fill: #c5c8c6 } -.terminal-1687321738-r2 { fill: #98a84b } -.terminal-1687321738-r3 { fill: #9a9b99 } -.terminal-1687321738-r4 { fill: #608ab1 } -.terminal-1687321738-r5 { fill: #d0b344 } -.terminal-1687321738-r6 { fill: #cc555a;font-weight: bold } -.terminal-1687321738-r7 { fill: #868887 } + .terminal-239277571-r1 { fill: #c5c8c6 } +.terminal-239277571-r2 { fill: #98a84b } +.terminal-239277571-r3 { fill: #9a9b99 } +.terminal-239277571-r4 { fill: #608ab1 } +.terminal-239277571-r5 { fill: #d0b344 } +.terminal-239277571-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -90,22 +89,22 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute           __main__.py:571 + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 68f27318a4..a2dc8099d4 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,65 +19,64 @@ font-weight: 700; } - .terminal-1533629559-matrix { + .terminal-3629854034-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1533629559-title { + .terminal-3629854034-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1533629559-r1 { fill: #c5c8c6 } -.terminal-1533629559-r2 { fill: #98a84b } -.terminal-1533629559-r3 { fill: #9a9b99 } -.terminal-1533629559-r4 { fill: #608ab1 } -.terminal-1533629559-r5 { fill: #d0b344 } -.terminal-1533629559-r6 { fill: #868887 } + .terminal-3629854034-r1 { fill: #c5c8c6 } +.terminal-3629854034-r2 { fill: #98a84b } +.terminal-3629854034-r3 { fill: #9a9b99 } +.terminal-3629854034-r4 { fill: #608ab1 } +.terminal-3629854034-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO     Removing abacas                                                                remove.py:56 + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index d815fd6787..92943a04a3 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,68 +19,67 @@ font-weight: 700; } - .terminal-566836186-matrix { + .terminal-53791440-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-566836186-title { + .terminal-53791440-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-566836186-r1 { fill: #c5c8c6 } -.terminal-566836186-r2 { fill: #98a84b } -.terminal-566836186-r3 { fill: #9a9b99 } -.terminal-566836186-r4 { fill: #608ab1 } -.terminal-566836186-r5 { fill: #d0b344 } -.terminal-566836186-r6 { fill: #868887 } + .terminal-53791440-r1 { fill: #c5c8c6 } +.terminal-53791440-r2 { fill: #98a84b } +.terminal-53791440-r3 { fill: #9a9b99 } +.terminal-53791440-r4 { fill: #608ab1 } +.terminal-53791440-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,23 +91,23 @@ - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view'module_test.py:180 + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view' diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 641fc97ac7..d1fa771510 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,77 +19,76 @@ font-weight: 700; } - .terminal-542272369-matrix { + .terminal-3977922886-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-542272369-title { + .terminal-3977922886-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-542272369-r1 { fill: #c5c8c6 } -.terminal-542272369-r2 { fill: #98a84b } -.terminal-542272369-r3 { fill: #9a9b99 } -.terminal-542272369-r4 { fill: #608ab1 } -.terminal-542272369-r5 { fill: #d0b344 } -.terminal-542272369-r6 { fill: #868887 } + .terminal-3977922886-r1 { fill: #c5c8c6 } +.terminal-3977922886-r2 { fill: #98a84b } +.terminal-3977922886-r3 { fill: #9a9b99 } +.terminal-3977922886-r4 { fill: #608ab1 } +.terminal-3977922886-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +100,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                update.py:165 -INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date           update.py:165 -INFO    'modules/nf-core/fastqc' is already up to date                                update.py:165 -INFO    'modules/nf-core/multiqc' is already up to date                               update.py:165 -INFO     Updates complete ✨                                                           update.py:247 + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO     Updating 'nf-core/custom/dumpsoftwareversions' +INFO     Updating 'nf-core/fastqc' +INFO    'modules/nf-core/multiqc' is already up to date                                             +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index db945fd020..90c012ed63 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-520878211-matrix { + .terminal-3325086355-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-520878211-title { + .terminal-3325086355-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-520878211-r1 { fill: #c5c8c6 } -.terminal-520878211-r2 { fill: #98a84b } -.terminal-520878211-r3 { fill: #9a9b99 } -.terminal-520878211-r4 { fill: #608ab1 } -.terminal-520878211-r5 { fill: #d0b344 } -.terminal-520878211-r6 { fill: #98a84b;font-weight: bold } -.terminal-520878211-r7 { fill: #868887 } -.terminal-520878211-r8 { fill: #868887;font-weight: bold } -.terminal-520878211-r9 { fill: #4e707b;font-weight: bold } -.terminal-520878211-r10 { fill: #68a0b3;font-weight: bold } + .terminal-3325086355-r1 { fill: #c5c8c6 } +.terminal-3325086355-r2 { fill: #98a84b } +.terminal-3325086355-r3 { fill: #9a9b99 } +.terminal-3325086355-r4 { fill: #608ab1 } +.terminal-3325086355-r5 { fill: #d0b344 } +.terminal-3325086355-r6 { fill: #98a84b;font-weight: bold } +.terminal-3325086355-r7 { fill: #868887;font-weight: bold } +.terminal-3325086355-r8 { fill: #868887 } +.terminal-3325086355-r9 { fill: #4e707b;font-weight: bold } +.terminal-3325086355-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 30 params)schema.py:95 -INFO     Writing schema with 31 params: './nextflow_schema.json'schema.py:173 + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) +INFO     Writing schema with 32 params: './nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index 179672a372..c7c6f38486 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-4182908040-matrix { + .terminal-617173835-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4182908040-title { + .terminal-617173835-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4182908040-r1 { fill: #c5c8c6 } -.terminal-4182908040-r2 { fill: #98a84b } -.terminal-4182908040-r3 { fill: #9a9b99 } -.terminal-4182908040-r4 { fill: #608ab1 } -.terminal-4182908040-r5 { fill: #d0b344 } -.terminal-4182908040-r6 { fill: #98a84b;font-weight: bold } -.terminal-4182908040-r7 { fill: #868887 } -.terminal-4182908040-r8 { fill: #868887;font-weight: bold } -.terminal-4182908040-r9 { fill: #4e707b;font-weight: bold } + .terminal-617173835-r1 { fill: #c5c8c6 } +.terminal-617173835-r2 { fill: #98a84b } +.terminal-617173835-r3 { fill: #9a9b99 } +.terminal-617173835-r4 { fill: #608ab1 } +.terminal-617173835-r5 { fill: #d0b344 } +.terminal-617173835-r6 { fill: #98a84b;font-weight: bold } +.terminal-617173835-r7 { fill: #868887;font-weight: bold } +.terminal-617173835-r8 { fill: #868887 } +.terminal-617173835-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 31 params)schema.py:95 + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 32 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 755794c92b..d0afb57106 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-2556505363-matrix { + .terminal-3174760220-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2556505363-title { + .terminal-3174760220-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2556505363-r1 { fill: #c5c8c6 } -.terminal-2556505363-r2 { fill: #98a84b } -.terminal-2556505363-r3 { fill: #9a9b99 } -.terminal-2556505363-r4 { fill: #608ab1 } -.terminal-2556505363-r5 { fill: #d0b344 } -.terminal-2556505363-r6 { fill: #98a84b;font-weight: bold } -.terminal-2556505363-r7 { fill: #868887 } -.terminal-2556505363-r8 { fill: #868887;font-weight: bold } -.terminal-2556505363-r9 { fill: #4e707b;font-weight: bold } + .terminal-3174760220-r1 { fill: #c5c8c6 } +.terminal-3174760220-r2 { fill: #98a84b } +.terminal-3174760220-r3 { fill: #9a9b99 } +.terminal-3174760220-r4 { fill: #608ab1 } +.terminal-3174760220-r5 { fill: #d0b344 } +.terminal-3174760220-r6 { fill: #98a84b;font-weight: bold } +.terminal-3174760220-r7 { fill: #868887;font-weight: bold } +.terminal-3174760220-r8 { fill: #868887 } +.terminal-3174760220-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validationschema.py:237 -INFO    [✓] Pipeline schema looks valid(found 93 params)schema.py:95 -INFO    [✓] Input parameters look validschema.py:213 + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg new file mode 100644 index 0000000000..20fc71e2e7 --- /dev/null +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. + + + + diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg new file mode 100644 index 0000000000..41c763ff7a --- /dev/null +++ b/docs/images/nf-core-subworkflows-create.svg @@ -0,0 +1,143 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot  --label process_low --meta  +--force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +Usage: nf-core subworkflows create [OPTIONS] subworkflow name                                       + +Try 'nf-core subworkflows create -h' for help. +╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ +│ No such option: --label Did you mean --help?                                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + + diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg new file mode 100644 index 0000000000..b3c6b6ee42 --- /dev/null +++ b/docs/images/nf-core-subworkflows-info.svg @@ -0,0 +1,465 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.11.0/x64/bin/nf-core:8 in <module>│ +││ +│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ +│❱ 8 │   sys.exit(run_nf_core())                                                                  │ +│9 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:105 in   │ +│run_nf_core│ +││ +│ 104 │   # Launch the click cli│ +│❱  105 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ +│ 106 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ +│__call__│ +││ +│1129 │   │   """Alias for :meth:`main`."""│ +│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ +│1131 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ +│ in main│ +││ +│20 │   │   try:                                                                                │ +│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ +│22 │   │   │   ifnot standalone_mode:                                                         │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ +││ +│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ +│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ +│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ +││ +│1403 │   │   ifself.callback isnotNone:                                                     │ +│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ +│1405 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ +││ +│ 759 │   │   │   with ctx:                                                                     │ +│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ +│ 761 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/decorators.py:26 in    │ +│new_func│ +││ +│ 25 │   defnew_func(*args, **kwargs):  # type: ignore│ +│❱  26 │   │   return f(get_current_context(), *args, **kwargs)                                   │ +│ 27 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:1087 in  │ +│info│ +││ +│1086 │   │   )                                                                                 │ +│❱ 1087 │   │   stdout.print(subworkflow_info.get_component_info())                               │ +│1088 │   except (UserWarning, LookupError) as e:                                               │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:1│ +│49 in get_component_info│ +││ +│148 │   │   │ +│❱ 149 │   │   returnself.generate_component_info_help()                                         │ +│150 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:2│ +│62 in generate_component_info_help│ +││ +│261 │   │   │   │   │   inputs_table.add_row(                                                  │ +│❱ 262 │   │   │   │   │   │   f"[orange1 on black] {key} [/][dim i] ({info['type']})",           │ +│263 │   │   │   │   │   │   Markdown(info["description"] if info["description"] else""),      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +TypeError: 'NoneType' object is not subscriptable + + + + diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg new file mode 100644 index 0000000000..cfb6a57f4c --- /dev/null +++ b/docs/images/nf-core-subworkflows-install.svg @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. + + + + diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg new file mode 100644 index 0000000000..10b3198e2d --- /dev/null +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +[..truncated..] + + + + diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg new file mode 100644 index 0000000000..966ce82802 --- /dev/null +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                            ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_dedup_stats_samtools_umitools            │ +│ bam_markduplicates_picard                    │ +│ bam_qc_picard                                │ +│ bam_rseqc                                    │ +│ bam_sort_stats_samtools                      │ +│ bam_stats_samtools                           │ +│ bcl_demultiplex                              │ +│ bed_scatter_bedtools                         │ +│ bedgraph_bedclip_bedgraphtobigwig            │ +[..truncated..] + + + + diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg new file mode 100644 index 0000000000..d6667f8e68 --- /dev/null +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. + + + + diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg new file mode 100644 index 0000000000..a407a2844d --- /dev/null +++ b/docs/images/nf-core-subworkflows-test.svg @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. + + + + diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg new file mode 100644 index 0000000000..c83f453da6 --- /dev/null +++ b/docs/images/nf-core-subworkflows-update.svg @@ -0,0 +1,167 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. + + + + diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index dcd7d1790b..c607247f3c 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,73 +19,72 @@ font-weight: 700; } - .terminal-3476133111-matrix { + .terminal-1725630925-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3476133111-title { + .terminal-1725630925-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3476133111-r1 { fill: #c5c8c6 } -.terminal-3476133111-r2 { fill: #98a84b } -.terminal-3476133111-r3 { fill: #9a9b99 } -.terminal-3476133111-r4 { fill: #608ab1 } -.terminal-3476133111-r5 { fill: #d0b344 } -.terminal-3476133111-r6 { fill: #98729f } -.terminal-3476133111-r7 { fill: #ff2c7a } -.terminal-3476133111-r8 { fill: #868887 } + .terminal-1725630925-r1 { fill: #c5c8c6 } +.terminal-1725630925-r2 { fill: #98a84b } +.terminal-1725630925-r3 { fill: #9a9b99 } +.terminal-1725630925-r4 { fill: #608ab1 } +.terminal-1725630925-r5 { fill: #d0b344 } +.terminal-1725630925-r6 { fill: #98729f } +.terminal-1725630925-r7 { fill: #ff2c7a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,24 +96,24 @@ - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.6 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthingsync.py:95 -INFO     Original pipeline repository branch is 'master'sync.py:149 -INFO     Deleting all files in 'TEMPLATE' branch                                         sync.py:205 -INFO     Making a new template pipeline using pipeline variables                         sync.py:223 + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' +INFO     Deleting all files in 'TEMPLATE' branch                                                     +INFO     Making a new template pipeline using pipeline variables                                     From f3195edeb7d0af3445fad353c4d4fb8628e1e040 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 13:14:39 +0100 Subject: [PATCH 762/854] fix new syntax form set_output in sync.yml --- .github/workflows/sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index b72e8ada6f..7844999dcd 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -27,7 +27,7 @@ jobs: else curl -O https://nf-co.re/pipeline_names.json fi - echo "name=matrix::$(cat pipeline_names.json)" >> $GITHUB_OUTPUT + echo "matrix==$(cat pipeline_names.json)" >> $GITHUB_OUTPUT sync: needs: get-pipelines From ebd0b2dbab2036ca563cb1aceb3c977254855010 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 13:23:18 +0100 Subject: [PATCH 763/854] bump to version 2.7.1 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 736e9600b2..a96b0db398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # nf-core/tools: Changelog +## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] + +- Patch release to fix pipeline sync ([#2110](https://github.com/nf-core/tools/pull/2110)) + ## [v2.7 - Mercury Eagle](https://github.com/nf-core/tools/releases/tag/2.7) - [2022-12-07] Another big release with lots of new features and bug fixes. Thanks to all contributors! diff --git a/setup.py b/setup.py index c2559d1776..046f5ddf76 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.7" +version = "2.7.1" with open("README.md") as f: readme = f.read() From 8c0c71885709845aaefb899c277ac0726a27a1dc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 13:30:30 +0100 Subject: [PATCH 764/854] fix typo --- .github/workflows/sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 7844999dcd..f3dec0e086 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -27,7 +27,7 @@ jobs: else curl -O https://nf-co.re/pipeline_names.json fi - echo "matrix==$(cat pipeline_names.json)" >> $GITHUB_OUTPUT + echo "matrix=$(cat pipeline_names.json)" >> $GITHUB_OUTPUT sync: needs: get-pipelines From eecf1c7079ef89e918a7493fb2228ac2fc63e5e0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 8 Dec 2022 13:07:18 +0000 Subject: [PATCH 765/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 88 ++-- docs/images/nf-core-create.svg | 126 ++--- docs/images/nf-core-download.svg | 86 ++-- docs/images/nf-core-launch-rnaseq.svg | 82 ++-- docs/images/nf-core-licences.svg | 130 +++--- docs/images/nf-core-lint.svg | 178 +++---- docs/images/nf-core-list-rna.svg | 118 ++--- docs/images/nf-core-list-stars.svg | 104 ++--- docs/images/nf-core-list.svg | 108 ++--- docs/images/nf-core-modules-bump-version.svg | 106 ++--- docs/images/nf-core-modules-create-test.svg | 98 ++-- docs/images/nf-core-modules-create.svg | 132 ++---- docs/images/nf-core-modules-info.svg | 206 ++++----- docs/images/nf-core-modules-install.svg | 86 ++-- docs/images/nf-core-modules-lint.svg | 112 ++--- docs/images/nf-core-modules-list-local.svg | 130 +++--- docs/images/nf-core-modules-list-remote.svg | 132 +++--- docs/images/nf-core-modules-mulled.svg | 78 ++-- docs/images/nf-core-modules-patch.svg | 72 +-- docs/images/nf-core-modules-remove.svg | 70 +-- docs/images/nf-core-modules-test.svg | 74 +-- docs/images/nf-core-modules-update.svg | 86 ++-- docs/images/nf-core-schema-build.svg | 84 ++-- docs/images/nf-core-schema-lint.svg | 78 ++-- docs/images/nf-core-schema-validate.svg | 82 ++-- .../nf-core-subworkflows-create-test.svg | 128 ++--- docs/images/nf-core-subworkflows-create.svg | 108 ++--- docs/images/nf-core-subworkflows-info.svg | 436 +++++++++--------- docs/images/nf-core-subworkflows-install.svg | 128 ++--- .../nf-core-subworkflows-list-local.svg | 130 +++--- .../nf-core-subworkflows-list-remote.svg | 132 +++--- docs/images/nf-core-subworkflows-remove.svg | 128 ++--- docs/images/nf-core-subworkflows-test.svg | 128 ++--- docs/images/nf-core-subworkflows-update.svg | 128 ++--- docs/images/nf-core-sync.svg | 82 ++-- 35 files changed, 2066 insertions(+), 2108 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index 0dc1616cb8..d63cc442c5 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,77 +19,77 @@ font-weight: 700; } - .terminal-456906266-matrix { + .terminal-2897532537-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-456906266-title { + .terminal-2897532537-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-456906266-r1 { fill: #c5c8c6 } -.terminal-456906266-r2 { fill: #98a84b } -.terminal-456906266-r3 { fill: #9a9b99 } -.terminal-456906266-r4 { fill: #608ab1 } -.terminal-456906266-r5 { fill: #d0b344 } -.terminal-456906266-r6 { fill: #cc555a } + .terminal-2897532537-r1 { fill: #c5c8c6 } +.terminal-2897532537-r2 { fill: #98a84b } +.terminal-2897532537-r3 { fill: #9a9b99 } +.terminal-2897532537-r4 { fill: #608ab1 } +.terminal-2897532537-r5 { fill: #d0b344 } +.terminal-2897532537-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +101,26 @@ - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1' -INFO     Updated version in 'nextflow.config' - - version         = '1.0dev' - + version = '1.1' - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1' +INFO     Updated version in 'nextflow.config' + - version         = '1.0dev' + + version = '1.1' + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 4c7e243fff..78343c2d34 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,104 +19,104 @@ font-weight: 700; } - .terminal-1507512876-matrix { + .terminal-3886862987-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1507512876-title { + .terminal-3886862987-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1507512876-r1 { fill: #c5c8c6 } -.terminal-1507512876-r2 { fill: #98a84b } -.terminal-1507512876-r3 { fill: #9a9b99 } -.terminal-1507512876-r4 { fill: #608ab1 } -.terminal-1507512876-r5 { fill: #d0b344 } -.terminal-1507512876-r6 { fill: #98729f } -.terminal-1507512876-r7 { fill: #ff2c7a } -.terminal-1507512876-r8 { fill: #98a84b;font-weight: bold } -.terminal-1507512876-r9 { fill: #1984e9;text-decoration: underline; } + .terminal-3886862987-r1 { fill: #c5c8c6 } +.terminal-3886862987-r2 { fill: #98a84b } +.terminal-3886862987-r3 { fill: #9a9b99 } +.terminal-3886862987-r4 { fill: #608ab1 } +.terminal-3886862987-r5 { fill: #d0b344 } +.terminal-3886862987-r6 { fill: #98729f } +.terminal-3886862987-r7 { fill: #ff2c7a } +.terminal-3886862987-r8 { fill: #98a84b;font-weight: bold } +.terminal-3886862987-r9 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -128,34 +128,34 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' -INFO     Initialising pipeline git repository                                                        -INFO     Done. Remember to add a remote and push to GitHub:                                          - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      -INFO    !!!!!! IMPORTANT !!!!!! - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' +INFO     Initialising pipeline git repository                                                        +INFO     Done. Remember to add a remote and push to GitHub:                                          + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      +INFO    !!!!!! IMPORTANT !!!!!! + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 6b19cc6433..4db5b62de9 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-3721474111-matrix { + .terminal-405352606-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3721474111-title { + .terminal-405352606-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3721474111-r1 { fill: #c5c8c6 } -.terminal-3721474111-r2 { fill: #98a84b } -.terminal-3721474111-r3 { fill: #9a9b99 } -.terminal-3721474111-r4 { fill: #608ab1 } -.terminal-3721474111-r5 { fill: #d0b344 } + .terminal-405352606-r1 { fill: #c5c8c6 } +.terminal-405352606-r2 { fill: #98a84b } +.terminal-405352606-r3 { fill: #9a9b99 } +.terminal-405352606-r4 { fill: #608ab1 } +.terminal-405352606-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq' -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                                                      -INFO     Downloading centralised configs from GitHub                                                 + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq' +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                                                      +INFO     Downloading centralised configs from GitHub                                                 diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index 55a70279d9..52cfc344e2 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-1175629223-matrix { + .terminal-3090263558-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1175629223-title { + .terminal-3090263558-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1175629223-r1 { fill: #c5c8c6 } -.terminal-1175629223-r2 { fill: #98a84b } -.terminal-1175629223-r3 { fill: #9a9b99 } -.terminal-1175629223-r4 { fill: #608ab1 } -.terminal-1175629223-r5 { fill: #d0b344 } -.terminal-1175629223-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1175629223-r7 { fill: #68a0b3;font-weight: bold } + .terminal-3090263558-r1 { fill: #c5c8c6 } +.terminal-3090263558-r2 { fill: #98a84b } +.terminal-3090263558-r3 { fill: #9a9b99 } +.terminal-3090263558-r4 { fill: #608ab1 } +.terminal-3090263558-r5 { fill: #d0b344 } +.terminal-3090263558-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3090263558-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      -         files or profiles                                                                           - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1) + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      +         files or profiles                                                                           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index db45603a7b..38d2266df9 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2422327830-matrix { + .terminal-3614362229-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2422327830-title { + .terminal-3614362229-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2422327830-r1 { fill: #c5c8c6 } -.terminal-2422327830-r2 { fill: #98a84b } -.terminal-2422327830-r3 { fill: #9a9b99 } -.terminal-2422327830-r4 { fill: #608ab1 } -.terminal-2422327830-r5 { fill: #d0b344 } -.terminal-2422327830-r6 { fill: #68a0b3;font-weight: bold } -.terminal-2422327830-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-3614362229-r1 { fill: #c5c8c6 } +.terminal-3614362229-r2 { fill: #98a84b } +.terminal-3614362229-r3 { fill: #9a9b99 } +.terminal-3614362229-r4 { fill: #608ab1 } +.terminal-3614362229-r5 { fill: #d0b344 } +.terminal-3614362229-r6 { fill: #68a0b3;font-weight: bold } +.terminal-3614362229-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    -INFO     Warning: This tool only prints licence information for the software tools packaged using    -         conda.                                                                                      -INFO     The pipeline may use other software and dependencies not described here.                    -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    +INFO     Warning: This tool only prints licence information for the software tools packaged using    +         conda.                                                                                      +INFO     The pipeline may use other software and dependencies not described here.                    +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index b255a7c4de..6ba942292b 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -19,142 +19,142 @@ font-weight: 700; } - .terminal-2054603030-matrix { + .terminal-2633876114-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2054603030-title { + .terminal-2633876114-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2054603030-r1 { fill: #c5c8c6 } -.terminal-2054603030-r2 { fill: #98a84b } -.terminal-2054603030-r3 { fill: #9a9b99 } -.terminal-2054603030-r4 { fill: #608ab1 } -.terminal-2054603030-r5 { fill: #d0b344 } -.terminal-2054603030-r6 { fill: #98729f } -.terminal-2054603030-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2054603030-r8 { fill: #1984e9 } -.terminal-2054603030-r9 { fill: #d0b344;font-weight: bold } -.terminal-2054603030-r10 { fill: #8d7b39 } -.terminal-2054603030-r11 { fill: #cc555a } + .terminal-2633876114-r1 { fill: #c5c8c6 } +.terminal-2633876114-r2 { fill: #98a84b } +.terminal-2633876114-r3 { fill: #9a9b99 } +.terminal-2633876114-r4 { fill: #608ab1 } +.terminal-2633876114-r5 { fill: #d0b344 } +.terminal-2633876114-r6 { fill: #98729f } +.terminal-2633876114-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2633876114-r8 { fill: #1984e9 } +.terminal-2633876114-r9 { fill: #d0b344;font-weight: bold } +.terminal-2633876114-r10 { fill: #8d7b39 } +.terminal-2633876114-r11 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -166,46 +166,46 @@ - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/custom/…│New version available│ -│fastqc│modules/nf-core/fastqc  │New version available│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 178 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   2 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/custom/…│New version available│ +│fastqc│modules/nf-core/fastqc  │New version available│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 178 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   2 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 0dfaae8459..40830b3176 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-1440873036-matrix { + .terminal-3770022571-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1440873036-title { + .terminal-3770022571-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1440873036-r1 { fill: #c5c8c6 } -.terminal-1440873036-r2 { fill: #98a84b } -.terminal-1440873036-r3 { fill: #9a9b99 } -.terminal-1440873036-r4 { fill: #608ab1 } -.terminal-1440873036-r5 { fill: #d0b344 } -.terminal-1440873036-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1440873036-r7 { fill: #868887 } + .terminal-3770022571-r1 { fill: #c5c8c6 } +.terminal-3770022571-r2 { fill: #98a84b } +.terminal-3770022571-r3 { fill: #9a9b99 } +.terminal-3770022571-r4 { fill: #608ab1 } +.terminal-3770022571-r5 { fill: #d0b344 } +.terminal-3770022571-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3770022571-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq             │    44 │          2.1.0 │ 1 months ago │           - │ -                   │ -│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ -│ rnafusion            │    83 │          2.1.0 │ 5 months ago │           - │ -                   │ -│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    24 │            dev │            - │           - │ -                   │ -│ lncpipe              │    23 │            dev │            - │           - │ -                   │ -│ scflow               │    14 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq             │    44 │          2.1.0 │ 1 months ago │           - │ -                   │ +│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ +│ rnafusion            │    83 │          2.1.0 │ 5 months ago │           - │ -                   │ +│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    24 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    14 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 80d81c4dcd..af68101512 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-2329934618-matrix { + .terminal-4159699833-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2329934618-title { + .terminal-4159699833-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2329934618-r1 { fill: #c5c8c6 } -.terminal-2329934618-r2 { fill: #98a84b } -.terminal-2329934618-r3 { fill: #9a9b99 } -.terminal-2329934618-r4 { fill: #608ab1 } -.terminal-2329934618-r5 { fill: #d0b344 } -.terminal-2329934618-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2329934618-r7 { fill: #868887 } -.terminal-2329934618-r8 { fill: #868887;font-style: italic; } + .terminal-4159699833-r1 { fill: #c5c8c6 } +.terminal-4159699833-r2 { fill: #98a84b } +.terminal-4159699833-r3 { fill: #9a9b99 } +.terminal-4159699833-r4 { fill: #608ab1 } +.terminal-4159699833-r5 { fill: #d0b344 } +.terminal-4159699833-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4159699833-r7 { fill: #868887 } +.terminal-4159699833-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ -│ sarek                │   210 │          3.1.1 │  2 weeks ago │           - │ -                   │ -│ chipseq              │   133 │          2.0.0 │ 2 months ago │           - │ -                   │ -│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ +│ sarek                │   210 │          3.1.1 │  2 weeks ago │           - │ -                   │ +│ chipseq              │   133 │          2.0.0 │ 2 months ago │           - │ -                   │ +│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index efd90f47f8..24bc71baf4 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-2923789563-matrix { + .terminal-3135274331-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2923789563-title { + .terminal-3135274331-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2923789563-r1 { fill: #c5c8c6 } -.terminal-2923789563-r2 { fill: #98a84b } -.terminal-2923789563-r3 { fill: #9a9b99 } -.terminal-2923789563-r4 { fill: #608ab1 } -.terminal-2923789563-r5 { fill: #d0b344 } -.terminal-2923789563-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2923789563-r7 { fill: #868887 } -.terminal-2923789563-r8 { fill: #868887;font-style: italic; } + .terminal-3135274331-r1 { fill: #c5c8c6 } +.terminal-3135274331-r2 { fill: #98a84b } +.terminal-3135274331-r3 { fill: #9a9b99 } +.terminal-3135274331-r4 { fill: #608ab1 } +.terminal-3135274331-r5 { fill: #d0b344 } +.terminal-3135274331-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3135274331-r7 { fill: #868887 } +.terminal-3135274331-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ ampliseq             │   101 │          2.4.1 │ 20 hours ago │           - │ -                   │ -│ airrflow             │    24 │          2.4.0 │   2 days ago │           - │ -                   │ -│ mhcquant             │    21 │          2.4.0 │   6 days ago │           - │ -                   │ -│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ -│ methylseq            │    95 │          2.2.0 │  1 weeks ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ ampliseq             │   101 │          2.4.1 │ 21 hours ago │           - │ -                   │ +│ airrflow             │    24 │          2.4.0 │   2 days ago │           - │ -                   │ +│ mhcquant             │    21 │          2.4.0 │   6 days ago │           - │ -                   │ +│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ +│ methylseq            │    95 │          2.2.0 │  1 weeks ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 9f760347a3..bdd738ba03 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-824833976-matrix { + .terminal-1795029015-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-824833976-title { + .terminal-1795029015-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-824833976-r1 { fill: #c5c8c6 } -.terminal-824833976-r2 { fill: #98a84b } -.terminal-824833976-r3 { fill: #9a9b99 } -.terminal-824833976-r4 { fill: #608ab1 } -.terminal-824833976-r5 { fill: #d0b344 } -.terminal-824833976-r6 { fill: #98a84b;font-weight: bold } -.terminal-824833976-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-1795029015-r1 { fill: #c5c8c6 } +.terminal-1795029015-r2 { fill: #98a84b } +.terminal-1795029015-r3 { fill: #9a9b99 } +.terminal-1795029015-r4 { fill: #608ab1 } +.terminal-1795029015-r5 { fill: #d0b344 } +.terminal-1795029015-r6 { fill: #98a84b;font-weight: bold } +.terminal-1795029015-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index a9bf5835bc..ab9e254f37 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-1793760250-matrix { + .terminal-1810277116-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1793760250-title { + .terminal-1810277116-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1793760250-r1 { fill: #c5c8c6 } -.terminal-1793760250-r2 { fill: #98a84b } -.terminal-1793760250-r3 { fill: #9a9b99 } -.terminal-1793760250-r4 { fill: #608ab1 } -.terminal-1793760250-r5 { fill: #d0b344 } -.terminal-1793760250-r6 { fill: #ff2c7a } -.terminal-1793760250-r7 { fill: #98729f } + .terminal-1810277116-r1 { fill: #c5c8c6 } +.terminal-1810277116-r2 { fill: #98a84b } +.terminal-1810277116-r3 { fill: #9a9b99 } +.terminal-1810277116-r4 { fill: #608ab1 } +.terminal-1810277116-r5 { fill: #d0b344 } +.terminal-1810277116-r6 { fill: #ff2c7a } +.terminal-1810277116-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpk231usbb -work-dir /tmp/tmpvt2u8pj3 + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc_single_end' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_single_end -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmplzj5q_cr -work-dir /tmp/tmpwkq3pr51 diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 90626c9a55..82c008269b 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9' -INFO     Using Docker container: 'quay.io/biocontainers/fastqc:0.11.9--hdfd78af_1' -INFO     Using Singularity container:                                                                -'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--hdfd78af_1' -INFO     Created / edited following files:                                                           -           ./modules/nf-core/fastqc/main.nf -           ./modules/nf-core/fastqc/meta.yml -           ./tests/modules/nf-core/fastqc/main.nf -           ./tests/modules/nf-core/fastqc/test.yml -           ./tests/modules/nf-core/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9' diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index b22069ca8c..b3f667942c 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-377483768-matrix { + .terminal-2882269783-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-377483768-title { + .terminal-2882269783-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-377483768-r1 { fill: #c5c8c6 } -.terminal-377483768-r2 { fill: #98a84b } -.terminal-377483768-r3 { fill: #9a9b99 } -.terminal-377483768-r4 { fill: #608ab1 } -.terminal-377483768-r5 { fill: #d0b344 } -.terminal-377483768-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-377483768-r7 { fill: #98a84b;font-weight: bold } -.terminal-377483768-r8 { fill: #868887 } -.terminal-377483768-r9 { fill: #d08442 } -.terminal-377483768-r10 { fill: #868887;font-style: italic; } -.terminal-377483768-r11 { fill: #98729f } + .terminal-2882269783-r1 { fill: #c5c8c6 } +.terminal-2882269783-r2 { fill: #98a84b } +.terminal-2882269783-r3 { fill: #9a9b99 } +.terminal-2882269783-r4 { fill: #608ab1 } +.terminal-2882269783-r5 { fill: #d0b344 } +.terminal-2882269783-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2882269783-r7 { fill: #98a84b;font-weight: bold } +.terminal-2882269783-r8 { fill: #868887 } +.terminal-2882269783-r9 { fill: #d08442 } +.terminal-2882269783-r10 { fill: #868887;font-style: italic; } +.terminal-2882269783-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 4a53bd06c8..fa6c7b941c 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-2071432195-matrix { + .terminal-3882716258-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2071432195-title { + .terminal-3882716258-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2071432195-r1 { fill: #c5c8c6 } -.terminal-2071432195-r2 { fill: #98a84b } -.terminal-2071432195-r3 { fill: #9a9b99 } -.terminal-2071432195-r4 { fill: #608ab1 } -.terminal-2071432195-r5 { fill: #d0b344 } + .terminal-3882716258-r1 { fill: #c5c8c6 } +.terminal-3882716258-r2 { fill: #98a84b } +.terminal-3882716258-r3 { fill: #9a9b99 } +.terminal-3882716258-r4 { fill: #608ab1 } +.terminal-3882716258-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Installing 'abacas' -INFO     Use the following statement to include this module:                                         - - include { ABACAS } from '../modules/nf-core/abacas/main'                                            - + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 0531cdfe79..f63cf77d39 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -19,94 +19,94 @@ font-weight: 700; } - .terminal-2882701330-matrix { + .terminal-3569125489-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2882701330-title { + .terminal-3569125489-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2882701330-r1 { fill: #c5c8c6 } -.terminal-2882701330-r2 { fill: #98a84b } -.terminal-2882701330-r3 { fill: #9a9b99 } -.terminal-2882701330-r4 { fill: #608ab1 } -.terminal-2882701330-r5 { fill: #d0b344 } -.terminal-2882701330-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2882701330-r7 { fill: #98a84b;font-weight: bold } -.terminal-2882701330-r8 { fill: #cc555a } + .terminal-3569125489-r1 { fill: #c5c8c6 } +.terminal-3569125489-r2 { fill: #98a84b } +.terminal-3569125489-r3 { fill: #9a9b99 } +.terminal-3569125489-r4 { fill: #608ab1 } +.terminal-3569125489-r5 { fill: #d0b344 } +.terminal-3569125489-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3569125489-r7 { fill: #98a84b;font-weight: bold } +.terminal-3569125489-r8 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -118,31 +118,31 @@ - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  23 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' + +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  23 Tests Passed │ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 0d5cdf6f2f..c2cbe2b236 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2639661206-matrix { + .terminal-801966325-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2639661206-title { + .terminal-801966325-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2639661206-r1 { fill: #c5c8c6 } -.terminal-2639661206-r2 { fill: #98a84b } -.terminal-2639661206-r3 { fill: #9a9b99 } -.terminal-2639661206-r4 { fill: #608ab1 } -.terminal-2639661206-r5 { fill: #d0b344 } -.terminal-2639661206-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2639661206-r7 { fill: #868887;font-style: italic; } + .terminal-801966325-r1 { fill: #c5c8c6 } +.terminal-801966325-r2 { fill: #98a84b } +.terminal-801966325-r3 { fill: #9a9b99 } +.terminal-801966325-r4 { fill: #608ab1 } +.terminal-801966325-r5 { fill: #d0b344 } +.terminal-801966325-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-801966325-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ +│                     │                    │                     │ nf-core/modules    │            │ +│                     │                    │                     │ repo (#2141)       │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 0f37822110..701b00db57 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-3342753500-matrix { + .terminal-3298778939-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3342753500-title { + .terminal-3298778939-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3342753500-r1 { fill: #c5c8c6 } -.terminal-3342753500-r2 { fill: #98a84b } -.terminal-3342753500-r3 { fill: #9a9b99 } -.terminal-3342753500-r4 { fill: #608ab1 } -.terminal-3342753500-r5 { fill: #d0b344 } -.terminal-3342753500-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-3342753500-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-3342753500-r8 { fill: #868887;font-style: italic; } + .terminal-3298778939-r1 { fill: #c5c8c6 } +.terminal-3298778939-r2 { fill: #98a84b } +.terminal-3298778939-r3 { fill: #9a9b99 } +.terminal-3298778939-r4 { fill: #608ab1 } +.terminal-3298778939-r5 { fill: #d0b344 } +.terminal-3298778939-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-3298778939-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3298778939-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):                     - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                                          ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                                │ -│ abricate/run                                          │ -│ abricate/summary                                      │ -│ adapterremoval                                        │ -│ adapterremovalfixprefix                               │ -│ agat/convertspgff2gtf                                 │ -│ agat/convertspgxf2gxf                                 │ -│ agat/spstatistics                                     │ -│ agat/sqstatbasic                                      │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ agat/convertspgff2gtf                                 │ +│ agat/convertspgxf2gxf                                 │ +│ agat/spstatistics                                     │ +│ agat/sqstatbasic                                      │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index 5373182936..b6ead66b4b 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -19,69 +19,69 @@ font-weight: 700; } - .terminal-3390485959-matrix { + .terminal-1123202598-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3390485959-title { + .terminal-1123202598-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3390485959-r1 { fill: #c5c8c6 } -.terminal-3390485959-r2 { fill: #98a84b } -.terminal-3390485959-r3 { fill: #9a9b99 } -.terminal-3390485959-r4 { fill: #608ab1 } -.terminal-3390485959-r5 { fill: #d0b344 } -.terminal-3390485959-r6 { fill: #00823d;font-weight: bold } -.terminal-3390485959-r7 { fill: #68a0b3;font-weight: bold } + .terminal-1123202598-r1 { fill: #c5c8c6 } +.terminal-1123202598-r2 { fill: #98a84b } +.terminal-1123202598-r3 { fill: #9a9b99 } +.terminal-1123202598-r4 { fill: #608ab1 } +.terminal-1123202598-r5 { fill: #d0b344 } +.terminal-1123202598-r6 { fill: #00823d;font-weight: bold } +.terminal-1123202598-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -93,23 +93,23 @@ - + - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                                           -INFO     Mulled container hash:                                                                      -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                                           +INFO     Mulled container hash:                                                                      +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 3bc3f80ffc..f54bff1e97 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-239277571-matrix { + .terminal-2789283426-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-239277571-title { + .terminal-2789283426-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-239277571-r1 { fill: #c5c8c6 } -.terminal-239277571-r2 { fill: #98a84b } -.terminal-239277571-r3 { fill: #9a9b99 } -.terminal-239277571-r4 { fill: #608ab1 } -.terminal-239277571-r5 { fill: #d0b344 } -.terminal-239277571-r6 { fill: #cc555a;font-weight: bold } + .terminal-2789283426-r1 { fill: #c5c8c6 } +.terminal-2789283426-r2 { fill: #98a84b } +.terminal-2789283426-r3 { fill: #9a9b99 } +.terminal-2789283426-r4 { fill: #608ab1 } +.terminal-2789283426-r5 { fill: #d0b344 } +.terminal-2789283426-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index a2dc8099d4..e495def404 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-3629854034-matrix { + .terminal-2407804337-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3629854034-title { + .terminal-2407804337-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3629854034-r1 { fill: #c5c8c6 } -.terminal-3629854034-r2 { fill: #98a84b } -.terminal-3629854034-r3 { fill: #9a9b99 } -.terminal-3629854034-r4 { fill: #608ab1 } -.terminal-3629854034-r5 { fill: #d0b344 } + .terminal-2407804337-r1 { fill: #c5c8c6 } +.terminal-2407804337-r2 { fill: #98a84b } +.terminal-2407804337-r3 { fill: #9a9b99 } +.terminal-2407804337-r4 { fill: #608ab1 } +.terminal-2407804337-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 92943a04a3..1ff788e24b 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,67 +19,67 @@ font-weight: 700; } - .terminal-53791440-matrix { + .terminal-367250223-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-53791440-title { + .terminal-367250223-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-53791440-r1 { fill: #c5c8c6 } -.terminal-53791440-r2 { fill: #98a84b } -.terminal-53791440-r3 { fill: #9a9b99 } -.terminal-53791440-r4 { fill: #608ab1 } -.terminal-53791440-r5 { fill: #d0b344 } + .terminal-367250223-r1 { fill: #c5c8c6 } +.terminal-367250223-r2 { fill: #98a84b } +.terminal-367250223-r3 { fill: #9a9b99 } +.terminal-367250223-r4 { fill: #608ab1 } +.terminal-367250223-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -91,23 +91,23 @@ - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view' + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view' diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index d1fa771510..34ee63f06e 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-3977922886-matrix { + .terminal-586434981-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3977922886-title { + .terminal-586434981-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3977922886-r1 { fill: #c5c8c6 } -.terminal-3977922886-r2 { fill: #98a84b } -.terminal-3977922886-r3 { fill: #9a9b99 } -.terminal-3977922886-r4 { fill: #608ab1 } -.terminal-3977922886-r5 { fill: #d0b344 } + .terminal-586434981-r1 { fill: #c5c8c6 } +.terminal-586434981-r2 { fill: #98a84b } +.terminal-586434981-r3 { fill: #9a9b99 } +.terminal-586434981-r4 { fill: #608ab1 } +.terminal-586434981-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO     Updating 'nf-core/custom/dumpsoftwareversions' -INFO     Updating 'nf-core/fastqc' -INFO    'modules/nf-core/multiqc' is already up to date                                             -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO     Updating 'nf-core/custom/dumpsoftwareversions' +INFO     Updating 'nf-core/fastqc' +INFO    'modules/nf-core/multiqc' is already up to date                                             +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 90c012ed63..91b1c4482f 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-3325086355-matrix { + .terminal-1763756786-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3325086355-title { + .terminal-1763756786-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3325086355-r1 { fill: #c5c8c6 } -.terminal-3325086355-r2 { fill: #98a84b } -.terminal-3325086355-r3 { fill: #9a9b99 } -.terminal-3325086355-r4 { fill: #608ab1 } -.terminal-3325086355-r5 { fill: #d0b344 } -.terminal-3325086355-r6 { fill: #98a84b;font-weight: bold } -.terminal-3325086355-r7 { fill: #868887;font-weight: bold } -.terminal-3325086355-r8 { fill: #868887 } -.terminal-3325086355-r9 { fill: #4e707b;font-weight: bold } -.terminal-3325086355-r10 { fill: #68a0b3;font-weight: bold } + .terminal-1763756786-r1 { fill: #c5c8c6 } +.terminal-1763756786-r2 { fill: #98a84b } +.terminal-1763756786-r3 { fill: #9a9b99 } +.terminal-1763756786-r4 { fill: #608ab1 } +.terminal-1763756786-r5 { fill: #d0b344 } +.terminal-1763756786-r6 { fill: #98a84b;font-weight: bold } +.terminal-1763756786-r7 { fill: #868887;font-weight: bold } +.terminal-1763756786-r8 { fill: #868887 } +.terminal-1763756786-r9 { fill: #4e707b;font-weight: bold } +.terminal-1763756786-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 31 params) -INFO     Writing schema with 32 params: './nextflow_schema.json' + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) +INFO     Writing schema with 32 params: './nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index c7c6f38486..9bb3f251e6 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-617173835-matrix { + .terminal-953635754-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-617173835-title { + .terminal-953635754-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-617173835-r1 { fill: #c5c8c6 } -.terminal-617173835-r2 { fill: #98a84b } -.terminal-617173835-r3 { fill: #9a9b99 } -.terminal-617173835-r4 { fill: #608ab1 } -.terminal-617173835-r5 { fill: #d0b344 } -.terminal-617173835-r6 { fill: #98a84b;font-weight: bold } -.terminal-617173835-r7 { fill: #868887;font-weight: bold } -.terminal-617173835-r8 { fill: #868887 } -.terminal-617173835-r9 { fill: #4e707b;font-weight: bold } + .terminal-953635754-r1 { fill: #c5c8c6 } +.terminal-953635754-r2 { fill: #98a84b } +.terminal-953635754-r3 { fill: #9a9b99 } +.terminal-953635754-r4 { fill: #608ab1 } +.terminal-953635754-r5 { fill: #d0b344 } +.terminal-953635754-r6 { fill: #98a84b;font-weight: bold } +.terminal-953635754-r7 { fill: #868887;font-weight: bold } +.terminal-953635754-r8 { fill: #868887 } +.terminal-953635754-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 32 params) + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 32 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index d0afb57106..8cc98e50b9 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-3174760220-matrix { + .terminal-2859597691-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3174760220-title { + .terminal-2859597691-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3174760220-r1 { fill: #c5c8c6 } -.terminal-3174760220-r2 { fill: #98a84b } -.terminal-3174760220-r3 { fill: #9a9b99 } -.terminal-3174760220-r4 { fill: #608ab1 } -.terminal-3174760220-r5 { fill: #d0b344 } -.terminal-3174760220-r6 { fill: #98a84b;font-weight: bold } -.terminal-3174760220-r7 { fill: #868887;font-weight: bold } -.terminal-3174760220-r8 { fill: #868887 } -.terminal-3174760220-r9 { fill: #4e707b;font-weight: bold } + .terminal-2859597691-r1 { fill: #c5c8c6 } +.terminal-2859597691-r2 { fill: #98a84b } +.terminal-2859597691-r3 { fill: #9a9b99 } +.terminal-2859597691-r4 { fill: #608ab1 } +.terminal-2859597691-r5 { fill: #d0b344 } +.terminal-2859597691-r6 { fill: #98a84b;font-weight: bold } +.terminal-2859597691-r7 { fill: #868887;font-weight: bold } +.terminal-2859597691-r8 { fill: #868887 } +.terminal-2859597691-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 93 params) -INFO    [✓] Input parameters look valid + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 20fc71e2e7..efe5f5bd5e 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-788073991-matrix { + .terminal-374083174-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-788073991-title { + .terminal-374083174-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-788073991-r1 { fill: #c5c8c6 } -.terminal-788073991-r2 { fill: #98a84b } -.terminal-788073991-r3 { fill: #9a9b99 } -.terminal-788073991-r4 { fill: #608ab1 } -.terminal-788073991-r5 { fill: #d0b344 } -.terminal-788073991-r6 { fill: #cc555a } + .terminal-374083174-r1 { fill: #c5c8c6 } +.terminal-374083174-r2 { fill: #98a84b } +.terminal-374083174-r3 { fill: #9a9b99 } +.terminal-374083174-r4 { fill: #608ab1 } +.terminal-374083174-r5 { fill: #d0b344 } +.terminal-374083174-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 41c763ff7a..9542f9758d 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-4205435415-matrix { + .terminal-2185943670-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4205435415-title { + .terminal-2185943670-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4205435415-r1 { fill: #c5c8c6 } -.terminal-4205435415-r2 { fill: #98a84b } -.terminal-4205435415-r3 { fill: #9a9b99 } -.terminal-4205435415-r4 { fill: #608ab1 } -.terminal-4205435415-r5 { fill: #d0b344 } -.terminal-4205435415-r6 { fill: #68a0b3;font-weight: bold } -.terminal-4205435415-r7 { fill: #868887 } -.terminal-4205435415-r8 { fill: #4a637a } -.terminal-4205435415-r9 { fill: #4a637a;font-weight: bold } -.terminal-4205435415-r10 { fill: #cc555a } + .terminal-2185943670-r1 { fill: #c5c8c6 } +.terminal-2185943670-r2 { fill: #98a84b } +.terminal-2185943670-r3 { fill: #9a9b99 } +.terminal-2185943670-r4 { fill: #608ab1 } +.terminal-2185943670-r5 { fill: #d0b344 } +.terminal-2185943670-r6 { fill: #68a0b3;font-weight: bold } +.terminal-2185943670-r7 { fill: #868887 } +.terminal-2185943670-r8 { fill: #4a637a } +.terminal-2185943670-r9 { fill: #4a637a;font-weight: bold } +.terminal-2185943670-r10 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,29 +114,29 @@ - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot  --label process_low --meta  ---force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -Usage: nf-core subworkflows create [OPTIONS] subworkflow name                                       - -Try 'nf-core subworkflows create -h' for help. -╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ -│ No such option: --label Did you mean --help?                                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot  --label process_low --meta  +--force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +Usage: nf-core subworkflows create [OPTIONS] subworkflow name                                       + +Try 'nf-core subworkflows create -h' for help. +╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ +│ No such option: --label Did you mean --help?                                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg index b3c6b6ee42..680dbfc817 100644 --- a/docs/images/nf-core-subworkflows-info.svg +++ b/docs/images/nf-core-subworkflows-info.svg @@ -19,333 +19,333 @@ font-weight: 700; } - .terminal-2903024194-matrix { + .terminal-1096786593-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2903024194-title { + .terminal-1096786593-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2903024194-r1 { fill: #c5c8c6 } -.terminal-2903024194-r2 { fill: #98a84b } -.terminal-2903024194-r3 { fill: #9a9b99 } -.terminal-2903024194-r4 { fill: #608ab1 } -.terminal-2903024194-r5 { fill: #d0b344 } -.terminal-2903024194-r6 { fill: #cc555a } -.terminal-2903024194-r7 { fill: #cc555a;font-weight: bold } -.terminal-2903024194-r8 { fill: #8a4346;font-weight: bold } -.terminal-2903024194-r9 { fill: #8d7b39 } -.terminal-2903024194-r10 { fill: #d0b344;font-weight: bold } -.terminal-2903024194-r11 { fill: #1984e9 } -.terminal-2903024194-r12 { fill: #00823d } -.terminal-2903024194-r13 { fill: #868887 } -.terminal-2903024194-r14 { fill: #398280 } -.terminal-2903024194-r15 { fill: #ff2c7a } -.terminal-2903024194-r16 { fill: #ff2627;font-weight: bold } + .terminal-1096786593-r1 { fill: #c5c8c6 } +.terminal-1096786593-r2 { fill: #98a84b } +.terminal-1096786593-r3 { fill: #9a9b99 } +.terminal-1096786593-r4 { fill: #608ab1 } +.terminal-1096786593-r5 { fill: #d0b344 } +.terminal-1096786593-r6 { fill: #cc555a } +.terminal-1096786593-r7 { fill: #cc555a;font-weight: bold } +.terminal-1096786593-r8 { fill: #8a4346;font-weight: bold } +.terminal-1096786593-r9 { fill: #8d7b39 } +.terminal-1096786593-r10 { fill: #d0b344;font-weight: bold } +.terminal-1096786593-r11 { fill: #1984e9 } +.terminal-1096786593-r12 { fill: #00823d } +.terminal-1096786593-r13 { fill: #868887 } +.terminal-1096786593-r14 { fill: #398280 } +.terminal-1096786593-r15 { fill: #ff2c7a } +.terminal-1096786593-r16 { fill: #ff2627;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -357,108 +357,108 @@ - + - - $ nf-core subworkflows info bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ -│/opt/hostedtoolcache/Python/3.11.0/x64/bin/nf-core:8 in <module>│ -││ -│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ -│❱ 8 │   sys.exit(run_nf_core())                                                                  │ -│9 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:105 in   │ -│run_nf_core│ -││ -│ 104 │   # Launch the click cli│ -│❱  105 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ -│ 106 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ -│__call__│ -││ -│1129 │   │   """Alias for :meth:`main`."""│ -│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ -│1131 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ -│ in main│ -││ -│20 │   │   try:                                                                                │ -│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ -│22 │   │   │   ifnot standalone_mode:                                                         │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ -││ -│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ -│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ -│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ -││ -│1403 │   │   ifself.callback isnotNone:                                                     │ -│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ -│1405 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ -││ -│ 759 │   │   │   with ctx:                                                                     │ -│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ -│ 761 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/decorators.py:26 in    │ -│new_func│ -││ -│ 25 │   defnew_func(*args, **kwargs):  # type: ignore│ -│❱  26 │   │   return f(get_current_context(), *args, **kwargs)                                   │ -│ 27 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:1087 in  │ -│info│ -││ -│1086 │   │   )                                                                                 │ -│❱ 1087 │   │   stdout.print(subworkflow_info.get_component_info())                               │ -│1088 │   except (UserWarning, LookupError) as e:                                               │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:1│ -│49 in get_component_info│ -││ -│148 │   │   │ -│❱ 149 │   │   returnself.generate_component_info_help()                                         │ -│150 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:2│ -│62 in generate_component_info_help│ -││ -│261 │   │   │   │   │   inputs_table.add_row(                                                  │ -│❱ 262 │   │   │   │   │   │   f"[orange1 on black] {key} [/][dim i] ({info['type']})",           │ -│263 │   │   │   │   │   │   Markdown(info["description"] if info["description"] else""),      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -TypeError: 'NoneType' object is not subscriptable + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.11.0/x64/bin/nf-core:8 in <module>│ +││ +│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ +│❱ 8 │   sys.exit(run_nf_core())                                                                  │ +│9 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:105 in   │ +│run_nf_core│ +││ +│ 104 │   # Launch the click cli│ +│❱  105 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ +│ 106 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ +│__call__│ +││ +│1129 │   │   """Alias for :meth:`main`."""│ +│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ +│1131 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ +│ in main│ +││ +│20 │   │   try:                                                                                │ +│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ +│22 │   │   │   ifnot standalone_mode:                                                         │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ +││ +│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ +│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ +│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ +││ +│1403 │   │   ifself.callback isnotNone:                                                     │ +│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ +│1405 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ +││ +│ 759 │   │   │   with ctx:                                                                     │ +│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ +│ 761 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/decorators.py:26 in    │ +│new_func│ +││ +│ 25 │   defnew_func(*args, **kwargs):  # type: ignore│ +│❱  26 │   │   return f(get_current_context(), *args, **kwargs)                                   │ +│ 27 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:1087 in  │ +│info│ +││ +│1086 │   │   )                                                                                 │ +│❱ 1087 │   │   stdout.print(subworkflow_info.get_component_info())                               │ +│1088 │   except (UserWarning, LookupError) as e:                                               │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:1│ +│49 in get_component_info│ +││ +│148 │   │   │ +│❱ 149 │   │   returnself.generate_component_info_help()                                         │ +│150 │ +││ +│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:2│ +│62 in generate_component_info_help│ +││ +│261 │   │   │   │   │   inputs_table.add_row(                                                  │ +│❱ 262 │   │   │   │   │   │   f"[orange1 on black] {key} [/][dim i] ({info['type']})",           │ +│263 │   │   │   │   │   │   Markdown(info["description"] if info["description"] else""),      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +TypeError: 'NoneType' object is not subscriptable diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg index cfb6a57f4c..4ad2943f2d 100644 --- a/docs/images/nf-core-subworkflows-install.svg +++ b/docs/images/nf-core-subworkflows-install.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-2034106394-matrix { + .terminal-1152843897-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2034106394-title { + .terminal-1152843897-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2034106394-r1 { fill: #c5c8c6 } -.terminal-2034106394-r2 { fill: #98a84b } -.terminal-2034106394-r3 { fill: #9a9b99 } -.terminal-2034106394-r4 { fill: #608ab1 } -.terminal-2034106394-r5 { fill: #d0b344 } -.terminal-2034106394-r6 { fill: #cc555a } + .terminal-1152843897-r1 { fill: #c5c8c6 } +.terminal-1152843897-r2 { fill: #98a84b } +.terminal-1152843897-r3 { fill: #9a9b99 } +.terminal-1152843897-r4 { fill: #608ab1 } +.terminal-1152843897-r5 { fill: #d0b344 } +.terminal-1152843897-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows install bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg index 10b3198e2d..3338be6182 100644 --- a/docs/images/nf-core-subworkflows-list-local.svg +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-1218770405-matrix { + .terminal-85784132-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1218770405-title { + .terminal-85784132-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1218770405-r1 { fill: #c5c8c6 } -.terminal-1218770405-r2 { fill: #98a84b } -.terminal-1218770405-r3 { fill: #9a9b99 } -.terminal-1218770405-r4 { fill: #608ab1 } -.terminal-1218770405-r5 { fill: #d0b344 } -.terminal-1218770405-r6 { fill: #cc555a } -.terminal-1218770405-r7 { fill: #868887;font-style: italic; } + .terminal-85784132-r1 { fill: #c5c8c6 } +.terminal-85784132-r2 { fill: #98a84b } +.terminal-85784132-r3 { fill: #9a9b99 } +.terminal-85784132-r4 { fill: #608ab1 } +.terminal-85784132-r5 { fill: #d0b344 } +.terminal-85784132-r6 { fill: #cc555a } +.terminal-85784132-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core subworkflows list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -[..truncated..] + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg index 966ce82802..9c60556afe 100644 --- a/docs/images/nf-core-subworkflows-list-remote.svg +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-649478967-matrix { + .terminal-4078322582-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-649478967-title { + .terminal-4078322582-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-649478967-r1 { fill: #c5c8c6 } -.terminal-649478967-r2 { fill: #98a84b } -.terminal-649478967-r3 { fill: #9a9b99 } -.terminal-649478967-r4 { fill: #608ab1 } -.terminal-649478967-r5 { fill: #d0b344 } -.terminal-649478967-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-649478967-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-649478967-r8 { fill: #868887;font-style: italic; } + .terminal-4078322582-r1 { fill: #c5c8c6 } +.terminal-4078322582-r2 { fill: #98a84b } +.terminal-4078322582-r3 { fill: #9a9b99 } +.terminal-4078322582-r4 { fill: #608ab1 } +.terminal-4078322582-r5 { fill: #d0b344 } +.terminal-4078322582-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-4078322582-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-4078322582-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core subworkflows list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Subworkflow Name                            ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bam_dedup_stats_samtools_umitools            │ -│ bam_markduplicates_picard                    │ -│ bam_qc_picard                                │ -│ bam_rseqc                                    │ -│ bam_sort_stats_samtools                      │ -│ bam_stats_samtools                           │ -│ bcl_demultiplex                              │ -│ bed_scatter_bedtools                         │ -│ bedgraph_bedclip_bedgraphtobigwig            │ -[..truncated..] + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                            ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_dedup_stats_samtools_umitools            │ +│ bam_markduplicates_picard                    │ +│ bam_qc_picard                                │ +│ bam_rseqc                                    │ +│ bam_sort_stats_samtools                      │ +│ bam_stats_samtools                           │ +│ bcl_demultiplex                              │ +│ bed_scatter_bedtools                         │ +│ bedgraph_bedclip_bedgraphtobigwig            │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index d6667f8e68..d6a95b7b77 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-1454309282-matrix { + .terminal-559218704-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1454309282-title { + .terminal-559218704-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1454309282-r1 { fill: #c5c8c6 } -.terminal-1454309282-r2 { fill: #98a84b } -.terminal-1454309282-r3 { fill: #9a9b99 } -.terminal-1454309282-r4 { fill: #608ab1 } -.terminal-1454309282-r5 { fill: #d0b344 } -.terminal-1454309282-r6 { fill: #cc555a } + .terminal-559218704-r1 { fill: #c5c8c6 } +.terminal-559218704-r2 { fill: #98a84b } +.terminal-559218704-r3 { fill: #9a9b99 } +.terminal-559218704-r4 { fill: #608ab1 } +.terminal-559218704-r5 { fill: #d0b344 } +.terminal-559218704-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index a407a2844d..0e3904fc3c 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-3187868540-matrix { + .terminal-2420114395-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3187868540-title { + .terminal-2420114395-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3187868540-r1 { fill: #c5c8c6 } -.terminal-3187868540-r2 { fill: #98a84b } -.terminal-3187868540-r3 { fill: #9a9b99 } -.terminal-3187868540-r4 { fill: #608ab1 } -.terminal-3187868540-r5 { fill: #d0b344 } -.terminal-3187868540-r6 { fill: #cc555a } + .terminal-2420114395-r1 { fill: #c5c8c6 } +.terminal-2420114395-r2 { fill: #98a84b } +.terminal-2420114395-r3 { fill: #9a9b99 } +.terminal-2420114395-r4 { fill: #608ab1 } +.terminal-2420114395-r5 { fill: #d0b344 } +.terminal-2420114395-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index c83f453da6..cee91fa471 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-2269839890-matrix { + .terminal-1454637681-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2269839890-title { + .terminal-1454637681-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2269839890-r1 { fill: #c5c8c6 } -.terminal-2269839890-r2 { fill: #98a84b } -.terminal-2269839890-r3 { fill: #9a9b99 } -.terminal-2269839890-r4 { fill: #608ab1 } -.terminal-2269839890-r5 { fill: #d0b344 } -.terminal-2269839890-r6 { fill: #cc555a } + .terminal-1454637681-r1 { fill: #c5c8c6 } +.terminal-1454637681-r2 { fill: #98a84b } +.terminal-1454637681-r3 { fill: #9a9b99 } +.terminal-1454637681-r4 { fill: #608ab1 } +.terminal-1454637681-r5 { fill: #d0b344 } +.terminal-1454637681-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index c607247f3c..2f22643569 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-1725630925-matrix { + .terminal-739183148-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1725630925-title { + .terminal-739183148-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1725630925-r1 { fill: #c5c8c6 } -.terminal-1725630925-r2 { fill: #98a84b } -.terminal-1725630925-r3 { fill: #9a9b99 } -.terminal-1725630925-r4 { fill: #608ab1 } -.terminal-1725630925-r5 { fill: #d0b344 } -.terminal-1725630925-r6 { fill: #98729f } -.terminal-1725630925-r7 { fill: #ff2c7a } + .terminal-739183148-r1 { fill: #c5c8c6 } +.terminal-739183148-r2 { fill: #98a84b } +.terminal-739183148-r3 { fill: #9a9b99 } +.terminal-739183148-r4 { fill: #608ab1 } +.terminal-739183148-r5 { fill: #d0b344 } +.terminal-739183148-r6 { fill: #98729f } +.terminal-739183148-r7 { fill: #ff2c7a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.1 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' +INFO     Deleting all files in 'TEMPLATE' branch                                                     +INFO     Making a new template pipeline using pipeline variables                                     From 51587c06811f113d2352f5d941bca1e355b6507c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 14:13:46 +0100 Subject: [PATCH 766/854] bump to v2.8dev --- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a96b0db398..7894ed3d65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # nf-core/tools: Changelog +# v2.8dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] - Patch release to fix pipeline sync ([#2110](https://github.com/nf-core/tools/pull/2110)) diff --git a/setup.py b/setup.py index 046f5ddf76..38f36810b7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.7.1" +version = "2.8dev" with open("README.md") as f: readme = f.read() From 46317bdcacd3321c23ed43e45345ee2e770d7f20 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 17:52:37 +0100 Subject: [PATCH 767/854] fix github_output syntax --- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 4 ++-- .../pipeline-template/.github/workflows/linting_comment.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 4c586a0929..f3dc3e50fe 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "name=result::pass" >> $GITHUB_OUTPUT + echo "result=pass" >> $GITHUB_OUTPUT else - echo "name=result::fail" >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 585b933f1c..09f8c423e5 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT + run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 From 78d4a25ee2d48c9177a3155c779a2699aa07d378 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Dec 2022 17:55:42 +0100 Subject: [PATCH 768/854] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7894ed3d65..fd09bbde70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) + ### Linting ### Modules From 0d7925bc1e7e0084b7956a0386b1cf7316d05f44 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Fri, 9 Dec 2022 14:35:19 +0000 Subject: [PATCH 769/854] Return from get_username and make sure it is assigned back to self.author --- nf_core/components/components_create.py | 2 ++ nf_core/modules/create.py | 2 +- nf_core/subworkflows/create.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py index a8d56411ba..86c42c0373 100644 --- a/nf_core/components/components_create.py +++ b/nf_core/components/components_create.py @@ -173,3 +173,5 @@ def get_username(author): f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", default=author_default, ) + + return author diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index b4899750d7..019a77c71f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -123,7 +123,7 @@ def create(self): self._get_bioconda_tool() # Prompt for GitHub username - nf_core.components.components_create.get_username(self.author) + self.author = nf_core.components.components_create.get_username(self.author) self._get_module_structure_components() diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index d5f9a5efbb..e61f0c6c8d 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -88,7 +88,7 @@ def create(self): ) # Prompt for GitHub username - nf_core.components.components_create.get_username(self.author) + self.author = nf_core.components.components_create.get_username(self.author) # Create subworkflow template with jinja2 nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) From 2cd002c218f81368a2789d61c171f148a66b4609 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Dec 2022 16:50:53 +0100 Subject: [PATCH 770/854] allow specifying only one container --- nf_core/modules/lint/main_nf.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c4f4bf9bc8..52aa30f7c3 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -271,6 +271,15 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) singularity_tag = None url = urlparse(l.split("'")[0]) + # lint double quotes + if l.count('"') > 2: + self.failed.append( + ( + "container_links", + "Too many double quotes found when specifying singularity container", + self.main_nf, + ) + ) if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 @@ -282,10 +291,14 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None url = urlparse(l.split("'")[0]) + # lint double quotes + if l.count('"') > 2: + self.failed.append( + ("container_links", "Too many double quotes found when specifying docker container", self.main_nf) + ) # lint double quotes if l.startswith("container"): - container_section = l + lines[i + 1] + lines[i + 2] - if container_section.count('"') > 2: + if l.count('"') > 2: self.failed.append( ("container_links", "Too many double quotes found when specifying containers", self.main_nf) ) From 96826eb00e832d51d543ddf78d805e7babb13cde Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Dec 2022 16:53:37 +0100 Subject: [PATCH 771/854] modify changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd09bbde70..f5584a9235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Linting +- Allow specifying containers in less than three lines ([#2121](https://github.com/nf-core/tools/pull/2121)) + ### Modules ### Subworkflows From a8cc7d9a10e01307c89e2f59af1cbba09c03ec69 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Fri, 9 Dec 2022 22:21:44 +0000 Subject: [PATCH 772/854] Ensure call to load_tools_config expects tuple return --- nf_core/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 3a5f1a502b..0bad1bec32 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -166,7 +166,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa param_dict["logo_dark"] = f"{param_dict['name_noslash']}_logo_dark.png" param_dict["version"] = version - config_yml = nf_core.utils.load_tools_config() + _, config_yml = nf_core.utils.load_tools_config() if ( "lint" in config_yml and "nextflow_config" in config_yml["lint"] From a85b8962b94bd5854424cc2d08d9c97ebf1a3982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 12 Dec 2022 13:41:09 +0000 Subject: [PATCH 773/854] check only one container per line --- nf_core/modules/lint/main_nf.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 52aa30f7c3..d44fe90f1e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -302,6 +302,15 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append( ("container_links", "Too many double quotes found when specifying containers", self.main_nf) ) + # lint more than one container in the same line + if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): + self.warned.append( + ( + "container_links", + "Docker and Singularity containers specified in the same line. Only first one checked.", + self.main_nf, + ) + ) # Try to connect to container URLs if url is None: continue From 6a7da79962da64c2e675a3efef012960a3299033 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 16:30:28 +0100 Subject: [PATCH 774/854] only check pipeline name without dashes if the name is provided by prompt --- nf_core/create.py | 7 ++++--- nf_core/sync.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 0bad1bec32..045c35d1b4 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -173,9 +173,10 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa and "manifest.name" in config_yml["lint"]["nextflow_config"] ): return param_dict, skip_paths - # Check that the pipeline name matches the requirements - if not re.match(r"^[a-z]+$", param_dict["short_name"]): - raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") + if param_dict["prefix"] == "nf-core": + # Check that the pipeline name matches the requirements + if not re.match(r"^[a-z]+$", param_dict["short_name"]): + raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") return param_dict, skip_paths diff --git a/nf_core/sync.py b/nf_core/sync.py index 4d0e1e88ba..332b524cbb 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -252,8 +252,9 @@ def make_template_pipeline(self): plain=True, ).init_pipeline() except Exception as err: - # If sync fails, remove template_yaml_path before raising error. - os.remove(self.template_yaml_path) + if self.template_yaml_path: + # If sync fails, remove template_yaml_path before raising error. + os.remove(self.template_yaml_path) # Reset to where you were to prevent git getting messed up. self.repo.git.reset("--hard") raise SyncException(f"Failed to rebuild pipeline from template with error:\n{err}") From f05b06e6f10bc7b971830aedeec2c827c5bd99ba Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 17:12:25 +0100 Subject: [PATCH 775/854] run prettier after dumping a json schema file --- nf_core/launch.py | 2 ++ nf_core/schema.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nf_core/launch.py b/nf_core/launch.py index d03e112c3e..ae4bb174ac 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -17,6 +17,7 @@ import nf_core.schema import nf_core.utils +from nf_core.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -703,6 +704,7 @@ def build_command(self): with open(self.params_out, "w") as fp: json.dump(self.schema_obj.input_params, fp, indent=4) fp.write("\n") + run_prettier_on_file(self.params_out) self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' # Call nextflow with a list of command line flags diff --git a/nf_core/schema.py b/nf_core/schema.py index d566f2b519..ee4a275c59 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -16,6 +16,7 @@ import nf_core.list import nf_core.utils +from nf_core.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -173,6 +174,7 @@ def save_schema(self, suppress_logging=False): with open(self.schema_filename, "w") as fh: json.dump(self.schema, fh, indent=4) fh.write("\n") + run_prettier_on_file(self.schema_filename) def load_input_params(self, params_path): """Load a given a path to a parameters file (JSON/YAML) From a0f5d568ce01c6f7ac6fa0f3dba9dad42afeab4d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 17:14:17 +0100 Subject: [PATCH 776/854] modify changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5584a9235..82acae6845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Linting - Allow specifying containers in less than three lines ([#2121](https://github.com/nf-core/tools/pull/2121)) +- Run prettier after dumping a json schema file ([#2124](https://github.com/nf-core/tools/pull/2124)) ### Modules From 5ccbae835e6b536a4d5490f6904bead1f971395a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Dec 2022 17:15:36 +0100 Subject: [PATCH 777/854] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5584a9235..ef0cc5e053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ ### General +- Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) + ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] - Patch release to fix pipeline sync ([#2110](https://github.com/nf-core/tools/pull/2110)) From e9e039a2679f487a861169caab147c5240c61b3d Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 13 Dec 2022 15:55:08 +0100 Subject: [PATCH 778/854] =?UTF-8?q?add=20newline=20before=20table=20?= =?UTF-8?q?=F0=9F=A4=9E=F0=9F=8F=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0ea83c3cca..8874837bc8 100644 --- a/README.md +++ b/README.md @@ -217,11 +217,12 @@ Please refer to the respective documentation for further details to manage packa ### Activate shell completions for nf-core/tools Auto-completion for the `nf-core` command is available for bash, zsh and fish. To activate it, add the following lines to the respective shell config files. -shell | shell config file | command ---- | --- | --- -bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` -Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` -fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` + +| shell | shell config file | command | +| ----- | --------------------------------------- | -------------------------------------------------- | +| bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | +| Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | +| fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options. From e3d987e2afaf76a0edc89c9a0ffb7e8fa970a220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 13 Dec 2022 16:04:08 +0100 Subject: [PATCH 779/854] Update README.md Co-authored-by: Phil Ewels --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8874837bc8..35a06028c9 100644 --- a/README.md +++ b/README.md @@ -220,9 +220,9 @@ Auto-completion for the `nf-core` command is available for bash, zsh and fish. T | shell | shell config file | command | | ----- | --------------------------------------- | -------------------------------------------------- | -| bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | -| Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | -| fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | +| bash | `~/.bashrc` | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | +| Zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | +| fish | `~/.config/fish/completions/nf-core.fish` | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options. From 5338afbb3a6e27f01a4a344eb3c566691a89ae73 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 13 Dec 2022 15:10:23 +0000 Subject: [PATCH 780/854] [automated] Fix code linting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35a06028c9..480b966f92 100644 --- a/README.md +++ b/README.md @@ -218,8 +218,8 @@ Please refer to the respective documentation for further details to manage packa Auto-completion for the `nf-core` command is available for bash, zsh and fish. To activate it, add the following lines to the respective shell config files. -| shell | shell config file | command | -| ----- | --------------------------------------- | -------------------------------------------------- | +| shell | shell config file | command | +| ----- | ----------------------------------------- | -------------------------------------------------- | | bash | `~/.bashrc` | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | | Zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | | fish | `~/.config/fish/completions/nf-core.fish` | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | From 6e7f87aae535110a0c29beadbca007418b9cd9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 13 Dec 2022 16:46:31 +0100 Subject: [PATCH 781/854] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 480b966f92..73d868bb32 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ Auto-completion for the `nf-core` command is available for bash, zsh and fish. T | shell | shell config file | command | | ----- | ----------------------------------------- | -------------------------------------------------- | | bash | `~/.bashrc` | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"` | -| Zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | +| zsh | `~/.zshrc` | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"` | | fish | `~/.config/fish/completions/nf-core.fish` | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)` | After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options. From 47b4252840817a1f2b77e26c816c6151157fd263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 13 Dec 2022 17:31:19 +0100 Subject: [PATCH 782/854] Apply suggestions from code review Co-authored-by: Phil Ewels --- nf_core/launch.py | 1 - nf_core/schema.py | 1 - 2 files changed, 2 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index ae4bb174ac..0facca72dc 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -703,7 +703,6 @@ def build_command(self): if self.use_params_file: with open(self.params_out, "w") as fp: json.dump(self.schema_obj.input_params, fp, indent=4) - fp.write("\n") run_prettier_on_file(self.params_out) self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' diff --git a/nf_core/schema.py b/nf_core/schema.py index ee4a275c59..92cbac852d 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -173,7 +173,6 @@ def save_schema(self, suppress_logging=False): log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") with open(self.schema_filename, "w") as fh: json.dump(self.schema, fh, indent=4) - fh.write("\n") run_prettier_on_file(self.schema_filename) def load_input_params(self, params_path): From 6eb466428600db57fa132ebfe28ffe0a0fe3f97c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Dec 2022 10:41:10 +0100 Subject: [PATCH 783/854] add json.dump into a helper function --- nf_core/launch.py | 6 ++---- nf_core/lint_utils.py | 12 ++++++++++++ nf_core/modules/modules_json.py | 7 ++----- nf_core/schema.py | 6 ++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 0facca72dc..87150172f7 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -17,7 +17,7 @@ import nf_core.schema import nf_core.utils -from nf_core.lint_utils import run_prettier_on_file +from nf_core.lint_utils import dump_json_with_prettier log = logging.getLogger(__name__) @@ -701,9 +701,7 @@ def build_command(self): # Write the user selection to a file and run nextflow with that if self.use_params_file: - with open(self.params_out, "w") as fp: - json.dump(self.schema_obj.input_params, fp, indent=4) - run_prettier_on_file(self.params_out) + dump_json_with_prettier(self.params_out, self.schema_obj.input_params) self.nextflow_cmd += f' -params-file "{os.path.relpath(self.params_out)}"' # Call nextflow with a list of command line flags diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index f8ca563b00..c2fd75d375 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -1,3 +1,4 @@ +import json import logging import subprocess from pathlib import Path @@ -82,3 +83,14 @@ def run_prettier_on_file(file): "There was an error running the prettier pre-commit hook.\n" f"STDOUT: {e.stdout.decode()}\nSTDERR: {e.stderr.decode()}" ) + + +def dump_json_with_prettier(file_name, file_content): + """Dump a JSON file and run prettier on it. + Args: + file_name (Path | str): A file identifier as a string or pathlib.Path. + file_content (dict): Content to dump into the JSON file + """ + with open(file_name, "w") as fh: + json.dump(file_content, fh, indent=4) + run_prettier_on_file(file_name) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cda7c827dc..a1609e5fee 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -13,7 +13,7 @@ import nf_core.utils from nf_core.components.components_utils import get_components_to_install -from nf_core.lint_utils import run_prettier_on_file +from nf_core.lint_utils import dump_json_with_prettier from nf_core.modules.modules_repo import ( NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, @@ -1023,10 +1023,7 @@ def dump(self): """ # Sort the modules.json self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - with open(self.modules_json_path, "w") as fh: - json.dump(self.modules_json, fh, indent=4) - fh.write("\n") - run_prettier_on_file(self.modules_json_path) + dump_json_with_prettier(self.modules_json_path, self.modules_json) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/schema.py b/nf_core/schema.py index 92cbac852d..04b17a9045 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -16,7 +16,7 @@ import nf_core.list import nf_core.utils -from nf_core.lint_utils import run_prettier_on_file +from nf_core.lint_utils import dump_json_with_prettier log = logging.getLogger(__name__) @@ -171,9 +171,7 @@ def save_schema(self, suppress_logging=False): num_params += sum(len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()) if not suppress_logging: log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") - with open(self.schema_filename, "w") as fh: - json.dump(self.schema, fh, indent=4) - run_prettier_on_file(self.schema_filename) + dump_json_with_prettier(self.schema_filename, self.schema) def load_input_params(self, params_path): """Load a given a path to a parameters file (JSON/YAML) From 433e5b2a9b532ccff164cf5e189bb32bf70a4a60 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 14:28:35 +0100 Subject: [PATCH 784/854] remove params.enable_conda --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 5 ----- nf_core/pipeline-template/nextflow.config | 3 --- nf_core/pipeline-template/nextflow_schema.json | 6 ------ 3 files changed, 14 deletions(-) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 714e659de9..c9c944b34a 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -75,11 +75,6 @@ class WorkflowMain { // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) - // Check that conda channels are set-up correctly - if (params.enable_conda) { - Utils.checkCondaChannels(log) - } - // Check AWS batch settings NfcoreTemplate.awsBatch(workflow, params) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2a7df54878..5cbb076b46 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -41,7 +41,6 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - enable_conda = false {% if nf_core_configs %} // Config options @@ -84,7 +83,6 @@ try { profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { - params.enable_conda = true conda.enabled = true docker.enabled = false singularity.enabled = false @@ -93,7 +91,6 @@ profiles { charliecloud.enabled = false } mamba { - params.enable_conda = true conda.enabled = true conda.useMamba = true docker.enabled = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 72c4e163a9..2743562d6c 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -263,12 +263,6 @@ "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "enable_conda": { - "type": "boolean", - "description": "Run this workflow with Conda. You can also use '-profile conda' instead of providing this parameter.", - "hidden": true, - "fa_icon": "fas fa-bacon" } } } From 547076bb0c16261a980e2d3abaacccd013e4e7df Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 14:50:46 +0100 Subject: [PATCH 785/854] add params.enable_conda to deprecated vars --- nf_core/lint/nextflow_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 0cf98a5b39..79bce3e7f1 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -146,6 +146,7 @@ def nextflow_config(self): "params.singleEnd", "params.igenomesIgnore", "params.name", + "params.enable_conda", ] # Remove field that should be ignored according to the linting config From d9be489acc55d40b419cb29670c731be376ba04a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 15:06:32 +0100 Subject: [PATCH 786/854] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c40fb3a035..28a49b70b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ### General - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) +- Deprecate `--enable_conda` parameter. Use `conda.enable` instead ([#2131](https://github.com/nf-core/tools/pull/2131)) ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] From 4cf1fdc3dcb59a0b67c35a17c8e47f586fb1cb70 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 15:25:10 +0100 Subject: [PATCH 787/854] update pipeline template modules --- nf_core/pipeline-template/modules.json | 6 +- .../custom/dumpsoftwareversions/main.nf | 2 +- .../templates/dumpsoftwareversions.py | 99 ++++++++++--------- .../modules/nf-core/fastqc/main.nf | 40 +++----- .../modules/nf-core/multiqc/main.nf | 2 +- 5 files changed, 76 insertions(+), 73 deletions(-) mode change 100644 => 100755 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 8618bacab6..08116ecbac 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] } } diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index cebb6e0589..3df21765b9 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,7 +2,7 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100644 new mode 100755 index 787bdb7b1b..e55b8d43a9 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,5 +1,9 @@ #!/usr/bin/env python + +"""Provide functions to merge multiple versions.yml files.""" + + import platform from textwrap import dedent @@ -7,6 +11,7 @@ def _make_versions_html(versions): + """Generate a tabular HTML output of all versions for MultiQC.""" html = [ dedent( """\\ @@ -45,47 +50,53 @@ def _make_versions_html(versions): return "\\n".join(html) -versions_this_module = {} -versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, -} - -with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - -# aggregate versions by the module name (derived from fully-qualified process name) -versions_by_module = {} -for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - -versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", -} - -versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), -} - -with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) -with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - -with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) +def main(): + """Load all version files and generate merged output.""" + versions_this_module = {} + versions_this_module["${task.process}"] = { + "python": platform.python_version(), + "yaml": yaml.__version__, + } + + with open("$versions") as f: + versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + + # aggregate versions by the module name (derived from fully-qualified process name) + versions_by_module = {} + for process, process_versions in versions_by_process.items(): + module = process.split(":")[-1] + try: + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) + except KeyError: + versions_by_module[module] = process_versions + + versions_by_module["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version", + } + + versions_mqc = { + "id": "software_versions", + "section_name": "${workflow.manifest.name} Software Versions", + "section_href": "https://github.com/${workflow.manifest.name}", + "plot_type": "html", + "description": "are collected at run time from the software output.", + "data": _make_versions_html(versions_by_module), + } + + with open("software_versions.yml", "w") as f: + yaml.dump(versions_by_module, f, default_flow_style=False) + with open("software_versions_mqc.yml", "w") as f: + yaml.dump(versions_mqc, f, default_flow_style=False) + + with open("versions.yml", "w") as f: + yaml.dump(versions_this_module, f, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 05730368b2..9ae5838158 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -2,7 +2,7 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : 'quay.io/biocontainers/fastqc:0.11.9--0' }" @@ -20,30 +20,22 @@ process FASTQC { script: def args = task.ext.args ?: '' - // Add soft-links to original FastQs for consistent naming in pipeline def prefix = task.ext.prefix ?: "${meta.id}" - if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $args --threads $task.cpus ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } + // Make list of old name and new name pairs to use for renaming in the bash while loop + def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } + def rename_to = old_new_pairs*.join(' ').join(' ') + def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + """ + printf "%s %s\\n" $rename_to | while read old_name new_name; do + [ -f "\${new_name}" ] || ln -s \$old_name \$new_name + done + fastqc $args --threads $task.cpus $renamed_files + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ stub: def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index a8159a57bf..68f66bea74 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -1,7 +1,7 @@ process MULTIQC { label 'process_single' - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" From c83965db55cf6a793c3525919b0fff05470af4c7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 15:49:55 +0100 Subject: [PATCH 788/854] fix test --- tests/modules/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/modules/update.py b/tests/modules/update.py index 19b53717fd..9729ef2975 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -329,6 +329,12 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): modules_json = ModulesJson(self.pipeline_dir) update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + # Update modules to a fixed old SHA + update_old = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, sha="5e34754d42cd2d5d248ca8673c0a53cdf5624905" + ) + update_old.update() + # Modify fastqc module, it will have a patch which will be applied during update # We modify fastqc because it's one of the modules that can be updated and there's another one before it (custom/dumpsoftwareversions) module_path = Path(self.pipeline_dir, "modules", "nf-core", "fastqc") From 3b2f53d1f6477ddef588ca6bdcc287253b7df626 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Dec 2022 16:29:17 +0100 Subject: [PATCH 789/854] fix test --- tests/modules/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/modules/update.py b/tests/modules/update.py index 9729ef2975..fcfd92fc39 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -303,6 +303,12 @@ def test_update_only_show_differences(self, mock_prompt): modules_json = ModulesJson(self.pipeline_dir) update_obj = ModuleUpdate(self.pipeline_dir, update_all=True, show_diff=True) + # Update modules to a fixed old SHA + update_old = ModuleUpdate( + self.pipeline_dir, update_all=True, show_diff=False, sha="5e34754d42cd2d5d248ca8673c0a53cdf5624905" + ) + update_old.update() + tmpdir = tempfile.mkdtemp() shutil.rmtree(tmpdir) shutil.copytree(Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME), tmpdir) From 8acc9d67922180486749175c193ccabfcb57c03d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 15 Dec 2022 23:42:46 +0100 Subject: [PATCH 790/854] Revert unique() syntax change. Closes #2132 --- CHANGELOG.md | 1 + nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c40fb3a035..47bf9e8f4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) +- Fix a bug introduced in 2.7 that made pipelines hang ([#2132](https://github.com/nf-core/tools/issues/2132)) ### Linting diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 2b54789101..9bcc0086b5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -82,7 +82,7 @@ workflow {{ short_name|upper }} { ch_versions = ch_versions.mix(FASTQC.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique{ it.text }.collectFile(name: 'collated_versions.yml') + ch_versions.unique().collectFile(name: 'collated_versions.yml') ) // From 8ce4d17bf3f24fff7175b0842ce5a2dca7b85884 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 09:49:00 +0100 Subject: [PATCH 791/854] handle json exception --- nf_core/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index b2431b3337..b60f61fff0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -248,7 +248,10 @@ def fetch_wf_config(wf_path, cache_config=True): if os.path.isfile(cache_path): log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path, "r") as fh: - config = json.load(fh) + try: + config = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{cache_path}' due to error {e}") return config log.debug("No config cache found") From 67c3656b60fd82c80a809198c0c621feb90d88bc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 09:52:54 +0100 Subject: [PATCH 792/854] handle all json-load exceptions --- nf_core/modules/modules_json.py | 6 +++++- nf_core/schema.py | 10 ++++++++-- tests/modules/modules_json.py | 10 ++++++++-- tests/test_launch.py | 6 +++++- tests/test_lint.py | 5 ++++- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index a1609e5fee..2d95afd10c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -618,7 +618,11 @@ def load(self): """ try: with open(self.modules_json_path, "r") as fh: - self.modules_json = json.load(fh) + try: + self.modules_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.modules_json_path}' due to error {e}") + except FileNotFoundError: raise UserWarning("File 'modules.json' is missing") diff --git a/nf_core/schema.py b/nf_core/schema.py index 04b17a9045..7ea9b972c0 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -105,7 +105,10 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" with open(self.schema_filename, "r") as fh: - self.schema = json.load(fh) + try: + self.schema = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.schema_filename}' due to error {e}") self.schema_defaults = {} self.schema_params = [] log.debug(f"JSON file loaded: {self.schema_filename}") @@ -182,7 +185,10 @@ def load_input_params(self, params_path): # First, try to load as JSON try: with open(params_path, "r") as fh: - params = json.load(fh) + try: + params = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{params_path}' due to error {e}") self.input_params.update(params) log.debug(f"Loaded JSON input params: {params_path}") except Exception as json_e: diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 67ee44f973..63ee4e743d 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -18,7 +18,10 @@ def test_get_modules_json(self): """Checks that the get_modules_json function returns the correct result""" mod_json_path = os.path.join(self.pipeline_dir, "modules.json") with open(mod_json_path, "r") as fh: - mod_json_sb = json.load(fh) + try: + mod_json_sb = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{mod_json_path}' due to error {e}") mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() @@ -212,7 +215,10 @@ def test_mod_json_dump(self): # Check that the dump function writes the correct content with open(mod_json_path, "r") as f: - mod_json_new = json.load(f) + try: + mod_json_new = json.load(f) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{mod_json_path}' due to error {e}") assert mod_json == mod_json_new diff --git a/tests/test_launch.py b/tests/test_launch.py index de597127a2..d830311ba3 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -327,7 +327,11 @@ def test_build_command_params(self): ) # Check saved parameters file with open(self.nf_params_fn, "r") as fh: - saved_json = json.load(fh) + try: + saved_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{self.nf_params_fn}' due to error {e}") + assert saved_json == {"input": "custom_input"} def test_build_command_params_cl(self): diff --git a/tests/test_lint.py b/tests/test_lint.py index 8160de9fbc..e4e93bd1f4 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -134,7 +134,10 @@ def test_json_output(self, tmp_dir): # Load created JSON file and check its contents with open(json_fn, "r") as fh: - saved_json = json.load(fh) + try: + saved_json = json.load(fh) + except json.JSONDecodeError as e: + raise UserWarning(f"Unable to load JSON file '{json_fn}' due to error {e}") assert saved_json["num_tests_pass"] > 0 assert saved_json["num_tests_warned"] > 0 assert saved_json["num_tests_ignored"] == 0 From 1625014620fd1202219369fa65b9daf6b8a49bad Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 09:54:36 +0100 Subject: [PATCH 793/854] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6842c3ffaa..ac0f8a6f6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) - Deprecate `--enable_conda` parameter. Use `conda.enable` instead ([#2131](https://github.com/nf-core/tools/pull/2131)) +- Handle `json.load()` exceptions ([#2134](https://github.com/nf-core/tools/pull/2134)) ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] From 632b675c15eed7c99fe9061c13be695347cf1c5f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 10:18:28 +0100 Subject: [PATCH 794/854] remove handling from linting as it's already handled later --- nf_core/schema.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 7ea9b972c0..a50a1ed789 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -105,10 +105,7 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" with open(self.schema_filename, "r") as fh: - try: - self.schema = json.load(fh) - except json.JSONDecodeError as e: - raise UserWarning(f"Unable to load JSON file '{self.schema_filename}' due to error {e}") + self.schema = json.load(fh) self.schema_defaults = {} self.schema_params = [] log.debug(f"JSON file loaded: {self.schema_filename}") From 0cb35e64dbec3ab94d7207ecad221f4072cade92 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 10:48:09 +0100 Subject: [PATCH 795/854] check conda channels as it's done in rnaseq --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index c9c944b34a..05db418b2d 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -75,6 +75,11 @@ class WorkflowMain { // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) + // Check that conda channels are set-up correctly + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + Utils.checkCondaChannels(log) + } + // Check AWS batch settings NfcoreTemplate.awsBatch(workflow, params) From 81fc96627bdc0af31bc186d2ef6acde7677e97fd Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 16 Dec 2022 10:50:07 +0100 Subject: [PATCH 796/854] remove params.enable_conda from modules template --- nf_core/module-template/modules/main.nf | 2 +- nf_core/pipeline-template/modules/local/samplesheet_check.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 6cfd5af2d5..e8f043f083 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -23,7 +23,7 @@ process {{ tool_name_underscore|upper }} { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - conda (params.enable_conda ? "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" : null) + conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'quay.io/biocontainers/YOUR-TOOL-HERE' }}' }" diff --git a/nf_core/pipeline-template/modules/local/samplesheet_check.nf b/nf_core/pipeline-template/modules/local/samplesheet_check.nf index 03a50c1040..5d25800775 100644 --- a/nf_core/pipeline-template/modules/local/samplesheet_check.nf +++ b/nf_core/pipeline-template/modules/local/samplesheet_check.nf @@ -2,7 +2,7 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" label 'process_single' - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : 'quay.io/biocontainers/python:3.8.3' }" From 02acf6561aae7a9b40d4bcd5ffb3fdc9da818dd5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 19 Dec 2022 09:45:03 +0100 Subject: [PATCH 797/854] bump to 2.7.2 for patch release --- CHANGELOG.md | 6 +----- setup.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0f8a6f6a..923a6c1ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -# v2.8dev +## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template @@ -12,10 +12,6 @@ - Allow specifying containers in less than three lines ([#2121](https://github.com/nf-core/tools/pull/2121)) - Run prettier after dumping a json schema file ([#2124](https://github.com/nf-core/tools/pull/2124)) -### Modules - -### Subworkflows - ### General - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) diff --git a/setup.py b/setup.py index 38f36810b7..cc83e90eed 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.8dev" +version = "2.7.2" with open("README.md") as f: readme = f.read() From 99961bedab1518f592668727a4d692c4ddf3c336 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 19 Dec 2022 11:44:18 +0000 Subject: [PATCH 798/854] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 88 ++-- docs/images/nf-core-create.svg | 126 ++--- docs/images/nf-core-download.svg | 86 ++-- docs/images/nf-core-launch-rnaseq.svg | 82 ++-- docs/images/nf-core-licences.svg | 130 +++--- docs/images/nf-core-lint.svg | 182 +++----- docs/images/nf-core-list-rna.svg | 118 ++--- docs/images/nf-core-list-stars.svg | 104 ++--- docs/images/nf-core-list.svg | 108 ++--- docs/images/nf-core-modules-bump-version.svg | 106 ++--- docs/images/nf-core-modules-create-test.svg | 98 ++-- docs/images/nf-core-modules-create.svg | 84 ++-- docs/images/nf-core-modules-info.svg | 206 ++++----- docs/images/nf-core-modules-install.svg | 86 ++-- docs/images/nf-core-modules-lint.svg | 112 ++--- docs/images/nf-core-modules-list-local.svg | 130 +++--- docs/images/nf-core-modules-list-remote.svg | 132 +++--- docs/images/nf-core-modules-mulled.svg | 78 ++-- docs/images/nf-core-modules-patch.svg | 72 +-- docs/images/nf-core-modules-remove.svg | 70 +-- docs/images/nf-core-modules-test.svg | 74 +-- docs/images/nf-core-modules-update.svg | 86 ++-- docs/images/nf-core-schema-build.svg | 84 ++-- docs/images/nf-core-schema-lint.svg | 78 ++-- docs/images/nf-core-schema-validate.svg | 82 ++-- .../nf-core-subworkflows-create-test.svg | 128 ++--- docs/images/nf-core-subworkflows-create.svg | 108 ++--- docs/images/nf-core-subworkflows-info.svg | 436 +++++++++--------- docs/images/nf-core-subworkflows-install.svg | 128 ++--- .../nf-core-subworkflows-list-local.svg | 130 +++--- .../nf-core-subworkflows-list-remote.svg | 132 +++--- docs/images/nf-core-subworkflows-remove.svg | 128 ++--- docs/images/nf-core-subworkflows-test.svg | 128 ++--- docs/images/nf-core-subworkflows-update.svg | 128 ++--- docs/images/nf-core-sync.svg | 82 ++-- 35 files changed, 2048 insertions(+), 2082 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index d63cc442c5..89843856b2 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,77 +19,77 @@ font-weight: 700; } - .terminal-2897532537-matrix { + .terminal-2980173434-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2897532537-title { + .terminal-2980173434-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2897532537-r1 { fill: #c5c8c6 } -.terminal-2897532537-r2 { fill: #98a84b } -.terminal-2897532537-r3 { fill: #9a9b99 } -.terminal-2897532537-r4 { fill: #608ab1 } -.terminal-2897532537-r5 { fill: #d0b344 } -.terminal-2897532537-r6 { fill: #cc555a } + .terminal-2980173434-r1 { fill: #c5c8c6 } +.terminal-2980173434-r2 { fill: #98a84b } +.terminal-2980173434-r3 { fill: #9a9b99 } +.terminal-2980173434-r4 { fill: #608ab1 } +.terminal-2980173434-r5 { fill: #d0b344 } +.terminal-2980173434-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101,26 +101,26 @@ - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1' -INFO     Updated version in 'nextflow.config' - - version         = '1.0dev' - + version = '1.1' - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1' +INFO     Updated version in 'nextflow.config' + - version         = '1.0dev' + + version = '1.1' + + diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 78343c2d34..17b28b0b67 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,104 +19,104 @@ font-weight: 700; } - .terminal-3886862987-matrix { + .terminal-4135113356-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3886862987-title { + .terminal-4135113356-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3886862987-r1 { fill: #c5c8c6 } -.terminal-3886862987-r2 { fill: #98a84b } -.terminal-3886862987-r3 { fill: #9a9b99 } -.terminal-3886862987-r4 { fill: #608ab1 } -.terminal-3886862987-r5 { fill: #d0b344 } -.terminal-3886862987-r6 { fill: #98729f } -.terminal-3886862987-r7 { fill: #ff2c7a } -.terminal-3886862987-r8 { fill: #98a84b;font-weight: bold } -.terminal-3886862987-r9 { fill: #1984e9;text-decoration: underline; } + .terminal-4135113356-r1 { fill: #c5c8c6 } +.terminal-4135113356-r2 { fill: #98a84b } +.terminal-4135113356-r3 { fill: #9a9b99 } +.terminal-4135113356-r4 { fill: #608ab1 } +.terminal-4135113356-r5 { fill: #d0b344 } +.terminal-4135113356-r6 { fill: #98729f } +.terminal-4135113356-r7 { fill: #ff2c7a } +.terminal-4135113356-r8 { fill: #98a84b;font-weight: bold } +.terminal-4135113356-r9 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -128,34 +128,34 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' -INFO     Initialising pipeline git repository                                                        -INFO     Done. Remember to add a remote and push to GitHub:                                          - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      -INFO    !!!!!! IMPORTANT !!!!!! - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' +INFO     Initialising pipeline git repository                                                        +INFO     Done. Remember to add a remote and push to GitHub:                                          + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      +INFO    !!!!!! IMPORTANT !!!!!! + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 4db5b62de9..d26c6011d5 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-405352606-matrix { + .terminal-512307359-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-405352606-title { + .terminal-512307359-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-405352606-r1 { fill: #c5c8c6 } -.terminal-405352606-r2 { fill: #98a84b } -.terminal-405352606-r3 { fill: #9a9b99 } -.terminal-405352606-r4 { fill: #608ab1 } -.terminal-405352606-r5 { fill: #d0b344 } + .terminal-512307359-r1 { fill: #c5c8c6 } +.terminal-512307359-r2 { fill: #98a84b } +.terminal-512307359-r3 { fill: #9a9b99 } +.terminal-512307359-r4 { fill: #608ab1 } +.terminal-512307359-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO     Saving 'nf-core/rnaseq' -          Pipeline revision: '3.8' -          Pull containers: 'none' -          Output directory: 'nf-core-rnaseq' -INFO     Downloading workflow files from GitHub                                                      -INFO     Downloading centralised configs from GitHub                                                 + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -c none + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO     Saving 'nf-core/rnaseq' +          Pipeline revision: '3.8' +          Pull containers: 'none' +          Output directory: 'nf-core-rnaseq' +INFO     Downloading workflow files from GitHub                                                      +INFO     Downloading centralised configs from GitHub                                                 diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index 52cfc344e2..c36c892e5a 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-3090263558-matrix { + .terminal-3166875143-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3090263558-title { + .terminal-3166875143-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3090263558-r1 { fill: #c5c8c6 } -.terminal-3090263558-r2 { fill: #98a84b } -.terminal-3090263558-r3 { fill: #9a9b99 } -.terminal-3090263558-r4 { fill: #608ab1 } -.terminal-3090263558-r5 { fill: #d0b344 } -.terminal-3090263558-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3090263558-r7 { fill: #68a0b3;font-weight: bold } + .terminal-3166875143-r1 { fill: #c5c8c6 } +.terminal-3166875143-r2 { fill: #98a84b } +.terminal-3166875143-r3 { fill: #9a9b99 } +.terminal-3166875143-r4 { fill: #608ab1 } +.terminal-3166875143-r5 { fill: #d0b344 } +.terminal-3166875143-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3166875143-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      -         files or profiles                                                                           - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1) + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      +         files or profiles                                                                           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 38d2266df9..f96e135487 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-3614362229-matrix { + .terminal-3773418102-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3614362229-title { + .terminal-3773418102-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3614362229-r1 { fill: #c5c8c6 } -.terminal-3614362229-r2 { fill: #98a84b } -.terminal-3614362229-r3 { fill: #9a9b99 } -.terminal-3614362229-r4 { fill: #608ab1 } -.terminal-3614362229-r5 { fill: #d0b344 } -.terminal-3614362229-r6 { fill: #68a0b3;font-weight: bold } -.terminal-3614362229-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-3773418102-r1 { fill: #c5c8c6 } +.terminal-3773418102-r2 { fill: #98a84b } +.terminal-3773418102-r3 { fill: #9a9b99 } +.terminal-3773418102-r4 { fill: #608ab1 } +.terminal-3773418102-r5 { fill: #d0b344 } +.terminal-3773418102-r6 { fill: #68a0b3;font-weight: bold } +.terminal-3773418102-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    -INFO     Warning: This tool only prints licence information for the software tools packaged using    -         conda.                                                                                      -INFO     The pipeline may use other software and dependencies not described here.                    -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    +INFO     Warning: This tool only prints licence information for the software tools packaged using    +         conda.                                                                                      +INFO     The pipeline may use other software and dependencies not described here.                    +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 6ba942292b..d375ae1f11 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/custom/…│New version available│ -│fastqc│modules/nf-core/fastqc  │New version available│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 178 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   2 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 181 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 40830b3176..e8346c16ca 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,99 +19,99 @@ font-weight: 700; } - .terminal-3770022571-matrix { + .terminal-3844799147-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3770022571-title { + .terminal-3844799147-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3770022571-r1 { fill: #c5c8c6 } -.terminal-3770022571-r2 { fill: #98a84b } -.terminal-3770022571-r3 { fill: #9a9b99 } -.terminal-3770022571-r4 { fill: #608ab1 } -.terminal-3770022571-r5 { fill: #d0b344 } -.terminal-3770022571-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3770022571-r7 { fill: #868887 } + .terminal-3844799147-r1 { fill: #c5c8c6 } +.terminal-3844799147-r2 { fill: #98a84b } +.terminal-3844799147-r3 { fill: #9a9b99 } +.terminal-3844799147-r4 { fill: #608ab1 } +.terminal-3844799147-r5 { fill: #d0b344 } +.terminal-3844799147-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3844799147-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -123,33 +123,33 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq             │    44 │          2.1.0 │ 1 months ago │           - │ -                   │ -│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ -│ rnafusion            │    83 │          2.1.0 │ 5 months ago │           - │ -                   │ -│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    24 │            dev │            - │           - │ -                   │ -│ lncpipe              │    23 │            dev │            - │           - │ -                   │ -│ scflow               │    14 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq             │    44 │          2.1.0 │ 2 months ago │           - │ -                   │ +│ rnaseq               │   542 │            3.9 │ 3 months ago │           - │ -                   │ +│ rnafusion            │    85 │          2.1.0 │ 5 months ago │           - │ -                   │ +│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ +│ circrna              │    23 │            dev │            - │           - │ -                   │ +│ lncpipe              │    23 │            dev │            - │           - │ -                   │ +│ scflow               │    15 │            dev │            - │           - │ -                   │ +│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index af68101512..57c4f1dba1 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-4159699833-matrix { + .terminal-1326709712-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4159699833-title { + .terminal-1326709712-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4159699833-r1 { fill: #c5c8c6 } -.terminal-4159699833-r2 { fill: #98a84b } -.terminal-4159699833-r3 { fill: #9a9b99 } -.terminal-4159699833-r4 { fill: #608ab1 } -.terminal-4159699833-r5 { fill: #d0b344 } -.terminal-4159699833-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4159699833-r7 { fill: #868887 } -.terminal-4159699833-r8 { fill: #868887;font-style: italic; } + .terminal-1326709712-r1 { fill: #c5c8c6 } +.terminal-1326709712-r2 { fill: #98a84b } +.terminal-1326709712-r3 { fill: #9a9b99 } +.terminal-1326709712-r4 { fill: #608ab1 } +.terminal-1326709712-r5 { fill: #d0b344 } +.terminal-1326709712-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1326709712-r7 { fill: #868887 } +.terminal-1326709712-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   538 │            3.9 │ 2 months ago │           - │ -                   │ -│ sarek                │   210 │          3.1.1 │  2 weeks ago │           - │ -                   │ -│ chipseq              │   133 │          2.0.0 │ 2 months ago │           - │ -                   │ -│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq               │   542 │            3.9 │ 3 months ago │           - │ -                   │ +│ sarek                │   211 │          3.1.1 │  4 weeks ago │           - │ -                   │ +│ chipseq              │   133 │          2.0.0 │ 3 months ago │           - │ -                   │ +│ atacseq              │   124 │            2.0 │  3 weeks ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 24bc71baf4..cf4bc6f125 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3135274331-matrix { + .terminal-4187913691-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3135274331-title { + .terminal-4187913691-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3135274331-r1 { fill: #c5c8c6 } -.terminal-3135274331-r2 { fill: #98a84b } -.terminal-3135274331-r3 { fill: #9a9b99 } -.terminal-3135274331-r4 { fill: #608ab1 } -.terminal-3135274331-r5 { fill: #d0b344 } -.terminal-3135274331-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3135274331-r7 { fill: #868887 } -.terminal-3135274331-r8 { fill: #868887;font-style: italic; } + .terminal-4187913691-r1 { fill: #c5c8c6 } +.terminal-4187913691-r2 { fill: #98a84b } +.terminal-4187913691-r3 { fill: #9a9b99 } +.terminal-4187913691-r4 { fill: #608ab1 } +.terminal-4187913691-r5 { fill: #d0b344 } +.terminal-4187913691-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4187913691-r7 { fill: #868887 } +.terminal-4187913691-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ ampliseq             │   101 │          2.4.1 │ 21 hours ago │           - │ -                   │ -│ airrflow             │    24 │          2.4.0 │   2 days ago │           - │ -                   │ -│ mhcquant             │    21 │          2.4.0 │   6 days ago │           - │ -                   │ -│ atacseq              │   123 │            2.0 │   1 week ago │           - │ -                   │ -│ methylseq            │    95 │          2.2.0 │  1 weeks ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ methylseq            │    95 │          2.3.0 │   2 days ago │           - │ -                   │ +│ ampliseq             │   102 │          2.4.1 │  2 weeks ago │           - │ -                   │ +│ airrflow             │    23 │          2.4.0 │  2 weeks ago │           - │ -                   │ +│ mhcquant             │    21 │          2.4.0 │  2 weeks ago │           - │ -                   │ +│ atacseq              │   124 │            2.0 │  3 weeks ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index bdd738ba03..227de6bb26 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1795029015-matrix { + .terminal-2041051160-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1795029015-title { + .terminal-2041051160-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1795029015-r1 { fill: #c5c8c6 } -.terminal-1795029015-r2 { fill: #98a84b } -.terminal-1795029015-r3 { fill: #9a9b99 } -.terminal-1795029015-r4 { fill: #608ab1 } -.terminal-1795029015-r5 { fill: #d0b344 } -.terminal-1795029015-r6 { fill: #98a84b;font-weight: bold } -.terminal-1795029015-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-2041051160-r1 { fill: #c5c8c6 } +.terminal-2041051160-r2 { fill: #98a84b } +.terminal-2041051160-r3 { fill: #9a9b99 } +.terminal-2041051160-r4 { fill: #608ab1 } +.terminal-2041051160-r5 { fill: #d0b344 } +.terminal-2041051160-r6 { fill: #98a84b;font-weight: bold } +.terminal-2041051160-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index ab9e254f37..57e29b961f 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-1810277116-matrix { + .terminal-474522378-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1810277116-title { + .terminal-474522378-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1810277116-r1 { fill: #c5c8c6 } -.terminal-1810277116-r2 { fill: #98a84b } -.terminal-1810277116-r3 { fill: #9a9b99 } -.terminal-1810277116-r4 { fill: #608ab1 } -.terminal-1810277116-r5 { fill: #d0b344 } -.terminal-1810277116-r6 { fill: #ff2c7a } -.terminal-1810277116-r7 { fill: #98729f } + .terminal-474522378-r1 { fill: #c5c8c6 } +.terminal-474522378-r2 { fill: #98a84b } +.terminal-474522378-r3 { fill: #9a9b99 } +.terminal-474522378-r4 { fill: #608ab1 } +.terminal-474522378-r5 { fill: #d0b344 } +.terminal-474522378-r6 { fill: #ff2c7a } +.terminal-474522378-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc_single_end' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_single_end -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmplzj5q_cr -work-dir /tmp/tmpwkq3pr51 + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc_single_end' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_single_end -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmp17isquh4 -work-dir /tmp/tmpseaglcoa diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 82c008269b..6b3ce27a41 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -19,74 +19,74 @@ font-weight: 700; } - .terminal-2925538580-matrix { + .terminal-3073846549-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2925538580-title { + .terminal-3073846549-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2925538580-r1 { fill: #c5c8c6 } -.terminal-2925538580-r2 { fill: #98a84b } -.terminal-2925538580-r3 { fill: #9a9b99 } -.terminal-2925538580-r4 { fill: #608ab1 } -.terminal-2925538580-r5 { fill: #d0b344 } -.terminal-2925538580-r6 { fill: #68a0b3;font-weight: bold } + .terminal-3073846549-r1 { fill: #c5c8c6 } +.terminal-3073846549-r2 { fill: #98a84b } +.terminal-3073846549-r3 { fill: #9a9b99 } +.terminal-3073846549-r4 { fill: #608ab1 } +.terminal-3073846549-r5 { fill: #d0b344 } +.terminal-3073846549-r6 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -98,25 +98,25 @@ - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9' + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9' diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index b3f667942c..320672f91a 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-2882269783-matrix { + .terminal-3462197848-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2882269783-title { + .terminal-3462197848-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2882269783-r1 { fill: #c5c8c6 } -.terminal-2882269783-r2 { fill: #98a84b } -.terminal-2882269783-r3 { fill: #9a9b99 } -.terminal-2882269783-r4 { fill: #608ab1 } -.terminal-2882269783-r5 { fill: #d0b344 } -.terminal-2882269783-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2882269783-r7 { fill: #98a84b;font-weight: bold } -.terminal-2882269783-r8 { fill: #868887 } -.terminal-2882269783-r9 { fill: #d08442 } -.terminal-2882269783-r10 { fill: #868887;font-style: italic; } -.terminal-2882269783-r11 { fill: #98729f } + .terminal-3462197848-r1 { fill: #c5c8c6 } +.terminal-3462197848-r2 { fill: #98a84b } +.terminal-3462197848-r3 { fill: #9a9b99 } +.terminal-3462197848-r4 { fill: #608ab1 } +.terminal-3462197848-r5 { fill: #d0b344 } +.terminal-3462197848-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3462197848-r7 { fill: #98a84b;font-weight: bold } +.terminal-3462197848-r8 { fill: #868887 } +.terminal-3462197848-r9 { fill: #d08442 } +.terminal-3462197848-r10 { fill: #868887;font-style: italic; } +.terminal-3462197848-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index fa6c7b941c..6fb029df0a 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-3882716258-matrix { + .terminal-3957689443-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3882716258-title { + .terminal-3957689443-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3882716258-r1 { fill: #c5c8c6 } -.terminal-3882716258-r2 { fill: #98a84b } -.terminal-3882716258-r3 { fill: #9a9b99 } -.terminal-3882716258-r4 { fill: #608ab1 } -.terminal-3882716258-r5 { fill: #d0b344 } + .terminal-3957689443-r1 { fill: #c5c8c6 } +.terminal-3957689443-r2 { fill: #98a84b } +.terminal-3957689443-r3 { fill: #9a9b99 } +.terminal-3957689443-r4 { fill: #608ab1 } +.terminal-3957689443-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Installing 'abacas' -INFO     Use the following statement to include this module:                                         - - include { ABACAS } from '../modules/nf-core/abacas/main'                                            - + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index f63cf77d39..b39f7aa4fd 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -19,94 +19,94 @@ font-weight: 700; } - .terminal-3569125489-matrix { + .terminal-3722872946-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3569125489-title { + .terminal-3722872946-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3569125489-r1 { fill: #c5c8c6 } -.terminal-3569125489-r2 { fill: #98a84b } -.terminal-3569125489-r3 { fill: #9a9b99 } -.terminal-3569125489-r4 { fill: #608ab1 } -.terminal-3569125489-r5 { fill: #d0b344 } -.terminal-3569125489-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3569125489-r7 { fill: #98a84b;font-weight: bold } -.terminal-3569125489-r8 { fill: #cc555a } + .terminal-3722872946-r1 { fill: #c5c8c6 } +.terminal-3722872946-r2 { fill: #98a84b } +.terminal-3722872946-r3 { fill: #9a9b99 } +.terminal-3722872946-r4 { fill: #608ab1 } +.terminal-3722872946-r5 { fill: #d0b344 } +.terminal-3722872946-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3722872946-r7 { fill: #98a84b;font-weight: bold } +.terminal-3722872946-r8 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -118,31 +118,31 @@ - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  23 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' + +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  23 Tests Passed │ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index c2cbe2b236..fab2cecf7e 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-801966325-matrix { + .terminal-2617511112-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-801966325-title { + .terminal-2617511112-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-801966325-r1 { fill: #c5c8c6 } -.terminal-801966325-r2 { fill: #98a84b } -.terminal-801966325-r3 { fill: #9a9b99 } -.terminal-801966325-r4 { fill: #608ab1 } -.terminal-801966325-r5 { fill: #d0b344 } -.terminal-801966325-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-801966325-r7 { fill: #868887;font-style: italic; } + .terminal-2617511112-r1 { fill: #c5c8c6 } +.terminal-2617511112-r2 { fill: #98a84b } +.terminal-2617511112-r3 { fill: #9a9b99 } +.terminal-2617511112-r4 { fill: #608ab1 } +.terminal-2617511112-r5 { fill: #d0b344 } +.terminal-2617511112-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2617511112-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ fastqc              │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -│ multiqc             │ https://github.co… │ 5e34754d42cd2d5d24… │ Restructure        │ 2022-10-04 │ -│                     │                    │                     │ nf-core/modules    │            │ -│                     │                    │                     │ repo (#2141)       │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +│ fastqc              │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +│ multiqc             │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 701b00db57..cd2d1df6e5 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-3298778939-matrix { + .terminal-3444989756-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3298778939-title { + .terminal-3444989756-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3298778939-r1 { fill: #c5c8c6 } -.terminal-3298778939-r2 { fill: #98a84b } -.terminal-3298778939-r3 { fill: #9a9b99 } -.terminal-3298778939-r4 { fill: #608ab1 } -.terminal-3298778939-r5 { fill: #d0b344 } -.terminal-3298778939-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-3298778939-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-3298778939-r8 { fill: #868887;font-style: italic; } + .terminal-3444989756-r1 { fill: #c5c8c6 } +.terminal-3444989756-r2 { fill: #98a84b } +.terminal-3444989756-r3 { fill: #9a9b99 } +.terminal-3444989756-r4 { fill: #608ab1 } +.terminal-3444989756-r5 { fill: #d0b344 } +.terminal-3444989756-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-3444989756-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3444989756-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):                     - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                                          ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                                │ -│ abricate/run                                          │ -│ abricate/summary                                      │ -│ adapterremoval                                        │ -│ adapterremovalfixprefix                               │ -│ agat/convertspgff2gtf                                 │ -│ agat/convertspgxf2gxf                                 │ -│ agat/spstatistics                                     │ -│ agat/sqstatbasic                                      │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ agat/convertspgff2gtf                                 │ +│ agat/convertspgxf2gxf                                 │ +│ agat/spstatistics                                     │ +│ agat/sqstatbasic                                      │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg index b6ead66b4b..2ac4b28f23 100644 --- a/docs/images/nf-core-modules-mulled.svg +++ b/docs/images/nf-core-modules-mulled.svg @@ -19,69 +19,69 @@ font-weight: 700; } - .terminal-1123202598-matrix { + .terminal-1198110247-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1123202598-title { + .terminal-1198110247-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1123202598-r1 { fill: #c5c8c6 } -.terminal-1123202598-r2 { fill: #98a84b } -.terminal-1123202598-r3 { fill: #9a9b99 } -.terminal-1123202598-r4 { fill: #608ab1 } -.terminal-1123202598-r5 { fill: #d0b344 } -.terminal-1123202598-r6 { fill: #00823d;font-weight: bold } -.terminal-1123202598-r7 { fill: #68a0b3;font-weight: bold } + .terminal-1198110247-r1 { fill: #c5c8c6 } +.terminal-1198110247-r2 { fill: #98a84b } +.terminal-1198110247-r3 { fill: #9a9b99 } +.terminal-1198110247-r4 { fill: #608ab1 } +.terminal-1198110247-r5 { fill: #d0b344 } +.terminal-1198110247-r6 { fill: #00823d;font-weight: bold } +.terminal-1198110247-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -93,23 +93,23 @@ - + - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                                           -INFO     Mulled container hash:                                                                      -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 + + $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO     Found docker image on quay.io! ✨                                                           +INFO     Mulled container hash:                                                                      +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index f54bff1e97..729bb1d519 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-2789283426-matrix { + .terminal-2827097699-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2789283426-title { + .terminal-2827097699-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2789283426-r1 { fill: #c5c8c6 } -.terminal-2789283426-r2 { fill: #98a84b } -.terminal-2789283426-r3 { fill: #9a9b99 } -.terminal-2789283426-r4 { fill: #608ab1 } -.terminal-2789283426-r5 { fill: #d0b344 } -.terminal-2789283426-r6 { fill: #cc555a;font-weight: bold } + .terminal-2827097699-r1 { fill: #c5c8c6 } +.terminal-2827097699-r2 { fill: #98a84b } +.terminal-2827097699-r3 { fill: #9a9b99 } +.terminal-2827097699-r4 { fill: #608ab1 } +.terminal-2827097699-r5 { fill: #d0b344 } +.terminal-2827097699-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index e495def404..ce72c00e13 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-2407804337-matrix { + .terminal-2450992562-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2407804337-title { + .terminal-2450992562-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2407804337-r1 { fill: #c5c8c6 } -.terminal-2407804337-r2 { fill: #98a84b } -.terminal-2407804337-r3 { fill: #9a9b99 } -.terminal-2407804337-r4 { fill: #608ab1 } -.terminal-2407804337-r5 { fill: #d0b344 } + .terminal-2450992562-r1 { fill: #c5c8c6 } +.terminal-2450992562-r2 { fill: #98a84b } +.terminal-2450992562-r3 { fill: #9a9b99 } +.terminal-2450992562-r4 { fill: #608ab1 } +.terminal-2450992562-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 1ff788e24b..c11314a7d1 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,67 +19,67 @@ font-weight: 700; } - .terminal-367250223-matrix { + .terminal-424201008-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-367250223-title { + .terminal-424201008-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-367250223-r1 { fill: #c5c8c6 } -.terminal-367250223-r2 { fill: #98a84b } -.terminal-367250223-r3 { fill: #9a9b99 } -.terminal-367250223-r4 { fill: #608ab1 } -.terminal-367250223-r5 { fill: #d0b344 } + .terminal-424201008-r1 { fill: #c5c8c6 } +.terminal-424201008-r2 { fill: #98a84b } +.terminal-424201008-r3 { fill: #9a9b99 } +.terminal-424201008-r4 { fill: #608ab1 } +.terminal-424201008-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -91,23 +91,23 @@ - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view' + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view' diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 34ee63f06e..161c6b2a56 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-586434981-matrix { + .terminal-555766662-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-586434981-title { + .terminal-555766662-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-586434981-r1 { fill: #c5c8c6 } -.terminal-586434981-r2 { fill: #98a84b } -.terminal-586434981-r3 { fill: #9a9b99 } -.terminal-586434981-r4 { fill: #608ab1 } -.terminal-586434981-r5 { fill: #d0b344 } + .terminal-555766662-r1 { fill: #c5c8c6 } +.terminal-555766662-r2 { fill: #98a84b } +.terminal-555766662-r3 { fill: #9a9b99 } +.terminal-555766662-r4 { fill: #608ab1 } +.terminal-555766662-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO     Updating 'nf-core/custom/dumpsoftwareversions' -INFO     Updating 'nf-core/fastqc' -INFO    'modules/nf-core/multiqc' is already up to date                                             -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date                         +INFO    'modules/nf-core/fastqc' is already up to date                                              +INFO    'modules/nf-core/multiqc' is already up to date                                             +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 91b1c4482f..ac95ebba84 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-1763756786-matrix { + .terminal-1833749233-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1763756786-title { + .terminal-1833749233-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1763756786-r1 { fill: #c5c8c6 } -.terminal-1763756786-r2 { fill: #98a84b } -.terminal-1763756786-r3 { fill: #9a9b99 } -.terminal-1763756786-r4 { fill: #608ab1 } -.terminal-1763756786-r5 { fill: #d0b344 } -.terminal-1763756786-r6 { fill: #98a84b;font-weight: bold } -.terminal-1763756786-r7 { fill: #868887;font-weight: bold } -.terminal-1763756786-r8 { fill: #868887 } -.terminal-1763756786-r9 { fill: #4e707b;font-weight: bold } -.terminal-1763756786-r10 { fill: #68a0b3;font-weight: bold } + .terminal-1833749233-r1 { fill: #c5c8c6 } +.terminal-1833749233-r2 { fill: #98a84b } +.terminal-1833749233-r3 { fill: #9a9b99 } +.terminal-1833749233-r4 { fill: #608ab1 } +.terminal-1833749233-r5 { fill: #d0b344 } +.terminal-1833749233-r6 { fill: #98a84b;font-weight: bold } +.terminal-1833749233-r7 { fill: #868887;font-weight: bold } +.terminal-1833749233-r8 { fill: #868887 } +.terminal-1833749233-r9 { fill: #4e707b;font-weight: bold } +.terminal-1833749233-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 31 params) -INFO     Writing schema with 32 params: './nextflow_schema.json' + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 30 params) +INFO     Writing schema with 31 params: './nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index 9bb3f251e6..79113e70ec 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-953635754-matrix { + .terminal-1041388458-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-953635754-title { + .terminal-1041388458-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-953635754-r1 { fill: #c5c8c6 } -.terminal-953635754-r2 { fill: #98a84b } -.terminal-953635754-r3 { fill: #9a9b99 } -.terminal-953635754-r4 { fill: #608ab1 } -.terminal-953635754-r5 { fill: #d0b344 } -.terminal-953635754-r6 { fill: #98a84b;font-weight: bold } -.terminal-953635754-r7 { fill: #868887;font-weight: bold } -.terminal-953635754-r8 { fill: #868887 } -.terminal-953635754-r9 { fill: #4e707b;font-weight: bold } + .terminal-1041388458-r1 { fill: #c5c8c6 } +.terminal-1041388458-r2 { fill: #98a84b } +.terminal-1041388458-r3 { fill: #9a9b99 } +.terminal-1041388458-r4 { fill: #608ab1 } +.terminal-1041388458-r5 { fill: #d0b344 } +.terminal-1041388458-r6 { fill: #98a84b;font-weight: bold } +.terminal-1041388458-r7 { fill: #868887;font-weight: bold } +.terminal-1041388458-r8 { fill: #868887 } +.terminal-1041388458-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 32 params) + + $ nf-core schema lint nextflow_schema.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 8cc98e50b9..3133ec66b8 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-2859597691-matrix { + .terminal-2998337404-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2859597691-title { + .terminal-2998337404-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2859597691-r1 { fill: #c5c8c6 } -.terminal-2859597691-r2 { fill: #98a84b } -.terminal-2859597691-r3 { fill: #9a9b99 } -.terminal-2859597691-r4 { fill: #608ab1 } -.terminal-2859597691-r5 { fill: #d0b344 } -.terminal-2859597691-r6 { fill: #98a84b;font-weight: bold } -.terminal-2859597691-r7 { fill: #868887;font-weight: bold } -.terminal-2859597691-r8 { fill: #868887 } -.terminal-2859597691-r9 { fill: #4e707b;font-weight: bold } + .terminal-2998337404-r1 { fill: #c5c8c6 } +.terminal-2998337404-r2 { fill: #98a84b } +.terminal-2998337404-r3 { fill: #9a9b99 } +.terminal-2998337404-r4 { fill: #608ab1 } +.terminal-2998337404-r5 { fill: #d0b344 } +.terminal-2998337404-r6 { fill: #98a84b;font-weight: bold } +.terminal-2998337404-r7 { fill: #868887;font-weight: bold } +.terminal-2998337404-r8 { fill: #868887 } +.terminal-2998337404-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 93 params) -INFO    [✓] Input parameters look valid + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index efe5f5bd5e..5e8841a50e 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-374083174-matrix { + .terminal-464653927-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-374083174-title { + .terminal-464653927-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-374083174-r1 { fill: #c5c8c6 } -.terminal-374083174-r2 { fill: #98a84b } -.terminal-374083174-r3 { fill: #9a9b99 } -.terminal-374083174-r4 { fill: #608ab1 } -.terminal-374083174-r5 { fill: #d0b344 } -.terminal-374083174-r6 { fill: #cc555a } + .terminal-464653927-r1 { fill: #c5c8c6 } +.terminal-464653927-r2 { fill: #98a84b } +.terminal-464653927-r3 { fill: #9a9b99 } +.terminal-464653927-r4 { fill: #608ab1 } +.terminal-464653927-r5 { fill: #d0b344 } +.terminal-464653927-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 9542f9758d..763a5a5d94 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-2185943670-matrix { + .terminal-2389301879-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2185943670-title { + .terminal-2389301879-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2185943670-r1 { fill: #c5c8c6 } -.terminal-2185943670-r2 { fill: #98a84b } -.terminal-2185943670-r3 { fill: #9a9b99 } -.terminal-2185943670-r4 { fill: #608ab1 } -.terminal-2185943670-r5 { fill: #d0b344 } -.terminal-2185943670-r6 { fill: #68a0b3;font-weight: bold } -.terminal-2185943670-r7 { fill: #868887 } -.terminal-2185943670-r8 { fill: #4a637a } -.terminal-2185943670-r9 { fill: #4a637a;font-weight: bold } -.terminal-2185943670-r10 { fill: #cc555a } + .terminal-2389301879-r1 { fill: #c5c8c6 } +.terminal-2389301879-r2 { fill: #98a84b } +.terminal-2389301879-r3 { fill: #9a9b99 } +.terminal-2389301879-r4 { fill: #608ab1 } +.terminal-2389301879-r5 { fill: #d0b344 } +.terminal-2389301879-r6 { fill: #68a0b3;font-weight: bold } +.terminal-2389301879-r7 { fill: #868887 } +.terminal-2389301879-r8 { fill: #4a637a } +.terminal-2389301879-r9 { fill: #4a637a;font-weight: bold } +.terminal-2389301879-r10 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,29 +114,29 @@ - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot  --label process_low --meta  ---force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -Usage: nf-core subworkflows create [OPTIONS] subworkflow name                                       - -Try 'nf-core subworkflows create -h' for help. -╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ -│ No such option: --label Did you mean --help?                                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot  --label process_low --meta  +--force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +Usage: nf-core subworkflows create [OPTIONS] subworkflow name                                       + +Try 'nf-core subworkflows create -h' for help. +╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ +│ No such option: --label Did you mean --help?                                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg index 680dbfc817..b00f918056 100644 --- a/docs/images/nf-core-subworkflows-info.svg +++ b/docs/images/nf-core-subworkflows-info.svg @@ -19,333 +19,333 @@ font-weight: 700; } - .terminal-1096786593-matrix { + .terminal-884777647-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1096786593-title { + .terminal-884777647-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1096786593-r1 { fill: #c5c8c6 } -.terminal-1096786593-r2 { fill: #98a84b } -.terminal-1096786593-r3 { fill: #9a9b99 } -.terminal-1096786593-r4 { fill: #608ab1 } -.terminal-1096786593-r5 { fill: #d0b344 } -.terminal-1096786593-r6 { fill: #cc555a } -.terminal-1096786593-r7 { fill: #cc555a;font-weight: bold } -.terminal-1096786593-r8 { fill: #8a4346;font-weight: bold } -.terminal-1096786593-r9 { fill: #8d7b39 } -.terminal-1096786593-r10 { fill: #d0b344;font-weight: bold } -.terminal-1096786593-r11 { fill: #1984e9 } -.terminal-1096786593-r12 { fill: #00823d } -.terminal-1096786593-r13 { fill: #868887 } -.terminal-1096786593-r14 { fill: #398280 } -.terminal-1096786593-r15 { fill: #ff2c7a } -.terminal-1096786593-r16 { fill: #ff2627;font-weight: bold } + .terminal-884777647-r1 { fill: #c5c8c6 } +.terminal-884777647-r2 { fill: #98a84b } +.terminal-884777647-r3 { fill: #9a9b99 } +.terminal-884777647-r4 { fill: #608ab1 } +.terminal-884777647-r5 { fill: #d0b344 } +.terminal-884777647-r6 { fill: #cc555a } +.terminal-884777647-r7 { fill: #cc555a;font-weight: bold } +.terminal-884777647-r8 { fill: #8a4346;font-weight: bold } +.terminal-884777647-r9 { fill: #8d7b39 } +.terminal-884777647-r10 { fill: #d0b344;font-weight: bold } +.terminal-884777647-r11 { fill: #1984e9 } +.terminal-884777647-r12 { fill: #00823d } +.terminal-884777647-r13 { fill: #868887 } +.terminal-884777647-r14 { fill: #398280 } +.terminal-884777647-r15 { fill: #ff2c7a } +.terminal-884777647-r16 { fill: #ff2627;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -357,108 +357,108 @@ - + - - $ nf-core subworkflows info bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ -│/opt/hostedtoolcache/Python/3.11.0/x64/bin/nf-core:8 in <module>│ -││ -│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ -│❱ 8 │   sys.exit(run_nf_core())                                                                  │ -│9 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:105 in   │ -│run_nf_core│ -││ -│ 104 │   # Launch the click cli│ -│❱  105 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ -│ 106 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ -│__call__│ -││ -│1129 │   │   """Alias for :meth:`main`."""│ -│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ -│1131 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ -│ in main│ -││ -│20 │   │   try:                                                                                │ -│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ -│22 │   │   │   ifnot standalone_mode:                                                         │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ -││ -│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ -│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ -│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ -││ -│1403 │   │   ifself.callback isnotNone:                                                     │ -│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ -│1405 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ -││ -│ 759 │   │   │   with ctx:                                                                     │ -│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ -│ 761 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/click/decorators.py:26 in    │ -│new_func│ -││ -│ 25 │   defnew_func(*args, **kwargs):  # type: ignore│ -│❱  26 │   │   return f(get_current_context(), *args, **kwargs)                                   │ -│ 27 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/__main__.py:1087 in  │ -│info│ -││ -│1086 │   │   )                                                                                 │ -│❱ 1087 │   │   stdout.print(subworkflow_info.get_component_info())                               │ -│1088 │   except (UserWarning, LookupError) as e:                                               │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:1│ -│49 in get_component_info│ -││ -│148 │   │   │ -│❱ 149 │   │   returnself.generate_component_info_help()                                         │ -│150 │ -││ -│/opt/hostedtoolcache/Python/3.11.0/x64/lib/python3.11/site-packages/nf_core/components/info.py:2│ -│62 in generate_component_info_help│ -││ -│261 │   │   │   │   │   inputs_table.add_row(                                                  │ -│❱ 262 │   │   │   │   │   │   f"[orange1 on black] {key} [/][dim i] ({info['type']})",           │ -│263 │   │   │   │   │   │   Markdown(info["description"] if info["description"] else""),      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -TypeError: 'NoneType' object is not subscriptable + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.11.1/x64/bin/nf-core:8 in <module>│ +││ +│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ +│❱ 8 │   sys.exit(run_nf_core())                                                                  │ +│9 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/__main__.py:105 in   │ +│run_nf_core│ +││ +│ 104 │   # Launch the click cli│ +│❱  105 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ +│ 106 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ +│__call__│ +││ +│1129 │   │   """Alias for :meth:`main`."""│ +│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ +│1131 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ +│ in main│ +││ +│20 │   │   try:                                                                                │ +│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ +│22 │   │   │   ifnot standalone_mode:                                                         │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ +││ +│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ +│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ +│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ +││ +│1656 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1658 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ +││ +│1403 │   │   ifself.callback isnotNone:                                                     │ +│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ +│1405 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ +││ +│ 759 │   │   │   with ctx:                                                                     │ +│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ +│ 761 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/decorators.py:26 in    │ +│new_func│ +││ +│ 25 │   defnew_func(*args, **kwargs):  # type: ignore│ +│❱  26 │   │   return f(get_current_context(), *args, **kwargs)                                   │ +│ 27 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/__main__.py:1087 in  │ +│info│ +││ +│1086 │   │   )                                                                                 │ +│❱ 1087 │   │   stdout.print(subworkflow_info.get_component_info())                               │ +│1088 │   except (UserWarning, LookupError) as e:                                               │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/components/info.py:1│ +│49 in get_component_info│ +││ +│148 │   │   │ +│❱ 149 │   │   returnself.generate_component_info_help()                                         │ +│150 │ +││ +│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/components/info.py:2│ +│62 in generate_component_info_help│ +││ +│261 │   │   │   │   │   inputs_table.add_row(                                                  │ +│❱ 262 │   │   │   │   │   │   f"[orange1 on black] {key} [/][dim i] ({info['type']})",           │ +│263 │   │   │   │   │   │   Markdown(info["description"] if info["description"] else""),      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +TypeError: 'NoneType' object is not subscriptable diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg index 4ad2943f2d..a440ce839e 100644 --- a/docs/images/nf-core-subworkflows-install.svg +++ b/docs/images/nf-core-subworkflows-install.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-1152843897-matrix { + .terminal-1243414650-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1152843897-title { + .terminal-1243414650-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1152843897-r1 { fill: #c5c8c6 } -.terminal-1152843897-r2 { fill: #98a84b } -.terminal-1152843897-r3 { fill: #9a9b99 } -.terminal-1152843897-r4 { fill: #608ab1 } -.terminal-1152843897-r5 { fill: #d0b344 } -.terminal-1152843897-r6 { fill: #cc555a } + .terminal-1243414650-r1 { fill: #c5c8c6 } +.terminal-1243414650-r2 { fill: #98a84b } +.terminal-1243414650-r3 { fill: #9a9b99 } +.terminal-1243414650-r4 { fill: #608ab1 } +.terminal-1243414650-r5 { fill: #d0b344 } +.terminal-1243414650-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows install bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg index 3338be6182..208e28e40e 100644 --- a/docs/images/nf-core-subworkflows-list-local.svg +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-85784132-matrix { + .terminal-174716485-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-85784132-title { + .terminal-174716485-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-85784132-r1 { fill: #c5c8c6 } -.terminal-85784132-r2 { fill: #98a84b } -.terminal-85784132-r3 { fill: #9a9b99 } -.terminal-85784132-r4 { fill: #608ab1 } -.terminal-85784132-r5 { fill: #d0b344 } -.terminal-85784132-r6 { fill: #cc555a } -.terminal-85784132-r7 { fill: #868887;font-style: italic; } + .terminal-174716485-r1 { fill: #c5c8c6 } +.terminal-174716485-r2 { fill: #98a84b } +.terminal-174716485-r3 { fill: #9a9b99 } +.terminal-174716485-r4 { fill: #608ab1 } +.terminal-174716485-r5 { fill: #d0b344 } +.terminal-174716485-r6 { fill: #cc555a } +.terminal-174716485-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core subworkflows list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -[..truncated..] + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg index 9c60556afe..4e2ff9f8a5 100644 --- a/docs/images/nf-core-subworkflows-list-remote.svg +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-4078322582-matrix { + .terminal-4215096215-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4078322582-title { + .terminal-4215096215-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4078322582-r1 { fill: #c5c8c6 } -.terminal-4078322582-r2 { fill: #98a84b } -.terminal-4078322582-r3 { fill: #9a9b99 } -.terminal-4078322582-r4 { fill: #608ab1 } -.terminal-4078322582-r5 { fill: #d0b344 } -.terminal-4078322582-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-4078322582-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-4078322582-r8 { fill: #868887;font-style: italic; } + .terminal-4215096215-r1 { fill: #c5c8c6 } +.terminal-4215096215-r2 { fill: #98a84b } +.terminal-4215096215-r3 { fill: #9a9b99 } +.terminal-4215096215-r4 { fill: #608ab1 } +.terminal-4215096215-r5 { fill: #d0b344 } +.terminal-4215096215-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-4215096215-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-4215096215-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core subworkflows list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Subworkflow Name                            ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bam_dedup_stats_samtools_umitools            │ -│ bam_markduplicates_picard                    │ -│ bam_qc_picard                                │ -│ bam_rseqc                                    │ -│ bam_sort_stats_samtools                      │ -│ bam_stats_samtools                           │ -│ bcl_demultiplex                              │ -│ bed_scatter_bedtools                         │ -│ bedgraph_bedclip_bedgraphtobigwig            │ -[..truncated..] + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                            ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_dedup_stats_samtools_umitools            │ +│ bam_markduplicates_picard                    │ +│ bam_qc_picard                                │ +│ bam_rseqc                                    │ +│ bam_sort_stats_samtools                      │ +│ bam_stats_samtools                           │ +│ bcl_demultiplex                              │ +│ bed_scatter_bedtools                         │ +│ bedgraph_bedclip_bedgraphtobigwig            │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index d6a95b7b77..33b93f5c19 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-559218704-matrix { + .terminal-649789457-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-559218704-title { + .terminal-649789457-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-559218704-r1 { fill: #c5c8c6 } -.terminal-559218704-r2 { fill: #98a84b } -.terminal-559218704-r3 { fill: #9a9b99 } -.terminal-559218704-r4 { fill: #608ab1 } -.terminal-559218704-r5 { fill: #d0b344 } -.terminal-559218704-r6 { fill: #cc555a } + .terminal-649789457-r1 { fill: #c5c8c6 } +.terminal-649789457-r2 { fill: #98a84b } +.terminal-649789457-r3 { fill: #9a9b99 } +.terminal-649789457-r4 { fill: #608ab1 } +.terminal-649789457-r5 { fill: #d0b344 } +.terminal-649789457-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 0e3904fc3c..44d0a5fc48 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-2420114395-matrix { + .terminal-2510685148-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2420114395-title { + .terminal-2510685148-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2420114395-r1 { fill: #c5c8c6 } -.terminal-2420114395-r2 { fill: #98a84b } -.terminal-2420114395-r3 { fill: #9a9b99 } -.terminal-2420114395-r4 { fill: #608ab1 } -.terminal-2420114395-r5 { fill: #d0b344 } -.terminal-2420114395-r6 { fill: #cc555a } + .terminal-2510685148-r1 { fill: #c5c8c6 } +.terminal-2510685148-r2 { fill: #98a84b } +.terminal-2510685148-r3 { fill: #9a9b99 } +.terminal-2510685148-r4 { fill: #608ab1 } +.terminal-2510685148-r5 { fill: #d0b344 } +.terminal-2510685148-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index cee91fa471..4769128e73 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,107 +19,107 @@ font-weight: 700; } - .terminal-1454637681-matrix { + .terminal-1545208434-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1454637681-title { + .terminal-1545208434-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1454637681-r1 { fill: #c5c8c6 } -.terminal-1454637681-r2 { fill: #98a84b } -.terminal-1454637681-r3 { fill: #9a9b99 } -.terminal-1454637681-r4 { fill: #608ab1 } -.terminal-1454637681-r5 { fill: #d0b344 } -.terminal-1454637681-r6 { fill: #cc555a } + .terminal-1545208434-r1 { fill: #c5c8c6 } +.terminal-1545208434-r2 { fill: #98a84b } +.terminal-1545208434-r3 { fill: #9a9b99 } +.terminal-1545208434-r4 { fill: #608ab1 } +.terminal-1545208434-r5 { fill: #d0b344 } +.terminal-1545208434-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -131,36 +131,36 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + + +WARNING 'repository_type' not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + » Pipeline +   nf-core/modules + + + +        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr +                                                                               o +w keys) + +Aborted. diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 2f22643569..99c164c146 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-739183148-matrix { + .terminal-832309805-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-739183148-title { + .terminal-832309805-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-739183148-r1 { fill: #c5c8c6 } -.terminal-739183148-r2 { fill: #98a84b } -.terminal-739183148-r3 { fill: #9a9b99 } -.terminal-739183148-r4 { fill: #608ab1 } -.terminal-739183148-r5 { fill: #d0b344 } -.terminal-739183148-r6 { fill: #98729f } -.terminal-739183148-r7 { fill: #ff2c7a } + .terminal-832309805-r1 { fill: #c5c8c6 } +.terminal-832309805-r2 { fill: #98a84b } +.terminal-832309805-r3 { fill: #9a9b99 } +.terminal-832309805-r4 { fill: #608ab1 } +.terminal-832309805-r5 { fill: #d0b344 } +.terminal-832309805-r6 { fill: #98729f } +.terminal-832309805-r7 { fill: #ff2c7a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.1 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.7.2 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' +INFO     Deleting all files in 'TEMPLATE' branch                                                     +INFO     Making a new template pipeline using pipeline variables                                     From a50d3520b5aaca09acf4be381f886697985210dc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 19 Dec 2022 13:11:30 +0100 Subject: [PATCH 799/854] bump to 2.8dev --- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 923a6c1ca4..d8c5d89f5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # nf-core/tools: Changelog +# v2.8dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template diff --git a/setup.py b/setup.py index cc83e90eed..38f36810b7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.7.2" +version = "2.8dev" with open("README.md") as f: readme = f.read() From 6777abb17d17da6887af548c3d00f0c326f4ff20 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 19 Dec 2022 13:35:03 +0100 Subject: [PATCH 800/854] change order in release checklist --- .github/RELEASE_CHECKLIST.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index c5b3464b83..f2d2591f16 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -8,8 +8,8 @@ 6. Create a PR from `dev` to `master` 7. Make sure all CI tests are passing again (additional tests are run on PRs to `master`) 8. Request review (2 approvals required) -9. Merge the PR into `master` -10. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button) +9. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button) +10. Merge the PR into `master` 11. Wait for CI tests on the commit to passed 12. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR. 13. Create a new release copying the `CHANGELOG` for that release into the description section. From 5d954788ebec7810bc0b77b173ecb2d58d403465 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 23 Dec 2022 10:30:45 +0100 Subject: [PATCH 801/854] Explicitly disable conda in container profiles --- nf_core/pipeline-template/nextflow.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 5cbb076b46..cc591a54e5 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -102,6 +102,7 @@ profiles { docker { docker.enabled = true docker.userEmulation = true + conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false @@ -113,6 +114,7 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + conda.enabled = false docker.enabled = false podman.enabled = false shifter.enabled = false @@ -120,6 +122,7 @@ profiles { } podman { podman.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false shifter.enabled = false @@ -127,6 +130,7 @@ profiles { } shifter { shifter.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false @@ -134,6 +138,7 @@ profiles { } charliecloud { charliecloud.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false From bb6ed953e593d4619aa25ceafba2074a464fe69d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 23 Dec 2022 10:32:17 +0100 Subject: [PATCH 802/854] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8c5d89f5d..62eef56c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) - Fix a bug introduced in 2.7 that made pipelines hang ([#2132](https://github.com/nf-core/tools/issues/2132)) +- Explicitly disable `conda` when using container profiles ([#2140](https://github.com/nf-core/tools/pull/2140)) ### Linting From a09c06099aecc9d88c5f9abe399fe810cde238d4 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 23 Dec 2022 10:35:00 +0100 Subject: [PATCH 803/854] Apply suggestions from code review --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62eef56c6c..be6d5820d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) - Fix a bug introduced in 2.7 that made pipelines hang ([#2132](https://github.com/nf-core/tools/issues/2132)) -- Explicitly disable `conda` when using container profiles ([#2140](https://github.com/nf-core/tools/pull/2140)) +- Explicitly disable `conda` when a container profile ([#2140](https://github.com/nf-core/tools/pull/2140)) ### Linting From bc9c44258733677db133809fd118fe6df502ba9e Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Sat, 7 Jan 2023 21:57:30 +0000 Subject: [PATCH 804/854] Make sure command runs when an output file is passed --- nf_core/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index aaf0ea9868..15eadc0484 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1446,8 +1446,9 @@ def docs(schema_path, output, format, force, columns): # Assume we're in a pipeline dir root if schema path not set schema_obj.get_schema_path(schema_path) schema_obj.load_schema() + docs = schema_obj.print_documentation(output, format, force, columns.split(",")) if not output: - stdout.print(schema_obj.print_documentation(output, format, force, columns.split(","))) + stdout.print(docs) # nf-core bump-version From ea52c3fbf62c7b9b281ce32bc1323a77da7347b2 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Sat, 7 Jan 2023 21:58:16 +0000 Subject: [PATCH 805/854] Replace newlines from fields in markdown tables with HTML
    as otherwise table formatting breaks --- nf_core/schema.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index a50a1ed789..d8046ecadd 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -495,9 +495,11 @@ def schema_to_markdown(self, columns): if column == "parameter": out += f"| `{p_key}` " elif column == "description": - out += f"| {param.get('description', '')} " + desc = param.get("description", "").replace("\n", "
    ") + out += f"| {desc} " if param.get("help_text", "") != "": - out += f"
    Help{param['help_text']}
    " + help_txt = param["help_text"].replace("\n", "
    ") + out += f"
    Help{help_txt}
    " elif column == "type": out += f"| `{param.get('type', '')}` " else: From e024fffb6d1bdaf6c72a274fef88feccfeb9390e Mon Sep 17 00:00:00 2001 From: Jose Alejandro Romero Herrera Date: Sun, 8 Jan 2023 21:05:36 +0100 Subject: [PATCH 806/854] Add singularity note for offline use --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 73d868bb32..ccb1d33378 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,8 @@ You can run the pipeline by simply providing the directory path for the `workflo nextflow run /path/to/download/nf-core-rnaseq-dev/workflow/ --input mydata.csv --outdir results # usual parameters here ``` +Note that you want to use the downloaded singularity images, you will still need to use `-profile singularity` or have it enabled in your config file. + ### Downloaded nf-core configs The pipeline files are automatically updated (`params.custom_config_base` is set to `../configs`), so that the local copy of institutional configs are available when running the pipeline. From b009559743de5732e0e82db416faad9c7e1e8d74 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 8 Jan 2023 21:48:22 +0100 Subject: [PATCH 807/854] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ccb1d33378..60dc102996 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,7 @@ You can run the pipeline by simply providing the directory path for the `workflo nextflow run /path/to/download/nf-core-rnaseq-dev/workflow/ --input mydata.csv --outdir results # usual parameters here ``` -Note that you want to use the downloaded singularity images, you will still need to use `-profile singularity` or have it enabled in your config file. +> Note that if you downloaded singularity images, you will need to use `-profile singularity` or have it enabled in your config file. ### Downloaded nf-core configs From a528fa8d96d9ba0ee80fb9ec5eb1e205b30c7246 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:32:09 +0000 Subject: [PATCH 808/854] Use technique from create_test_yml command to write prettified docs to file or console --- nf_core/__main__.py | 5 +---- nf_core/schema.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 15eadc0484..b96920b533 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1446,10 +1446,7 @@ def docs(schema_path, output, format, force, columns): # Assume we're in a pipeline dir root if schema path not set schema_obj.get_schema_path(schema_path) schema_obj.load_schema() - docs = schema_obj.print_documentation(output, format, force, columns.split(",")) - if not output: - stdout.print(docs) - + schema_obj.print_documentation(output, format, force, columns.split(",")) # nf-core bump-version @nf_core_cli.command("bump-version") diff --git a/nf_core/schema.py b/nf_core/schema.py index d8046ecadd..b325a6a075 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -6,17 +6,20 @@ import json import logging import os +import tempfile import webbrowser import jinja2 import jsonschema import markdown +import rich.console import yaml from rich.prompt import Confirm +from rich.syntax import Syntax import nf_core.list import nf_core.utils -from nf_core.lint_utils import dump_json_with_prettier +from nf_core.lint_utils import dump_json_with_prettier, run_prettier_on_file log = logging.getLogger(__name__) @@ -464,13 +467,21 @@ def print_documentation( if format == "html": output = self.markdown_to_html(output) - # Print to file - if output_fn: + with tempfile.NamedTemporaryFile(mode="w+") as fh: + fh.write(output) + run_prettier_on_file(fh.name) + fh.seek(0) + prettified_docs = fh.read() + + if not output_fn: + console = rich.console.Console() + console.print("\n", Syntax(prettified_docs, format), "\n") + else: if os.path.exists(output_fn) and not force: log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") return - with open(output_fn, "w") as file: - file.write(output) + with open(output_fn, "w") as fh: + fh.write(prettified_docs) log.info(f"Documentation written to '{output_fn}'") # Return as a string From c090d826024c12b15cbd858df7e91c450b0bdf4f Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:35:05 +0000 Subject: [PATCH 809/854] Fix black formatting --- nf_core/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b96920b533..555783d8d4 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1448,6 +1448,7 @@ def docs(schema_path, output, format, force, columns): schema_obj.load_schema() schema_obj.print_documentation(output, format, force, columns.split(",")) + # nf-core bump-version @nf_core_cli.command("bump-version") @click.argument("new_version", required=True, metavar="") From e32bd8c603388b2c29a30001db08a8573a93502b Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:32:52 +0000 Subject: [PATCH 810/854] Fix schema docs to correctly display required field --- nf_core/schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/schema.py b/nf_core/schema.py index b325a6a075..1d36d45c12 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -495,6 +495,7 @@ def schema_to_markdown(self, columns): out += f"{self.schema['description']}\n" # Grouped parameters for definition in self.schema.get("definitions", {}).values(): + required = definition.get("required", []) out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" out += "".join([f"| {column.title()} " for column in columns]) @@ -513,12 +514,15 @@ def schema_to_markdown(self, columns): out += f"
    Help{help_txt}
    " elif column == "type": out += f"| `{param.get('type', '')}` " + elif column == "required": + out += f"| {p_key in required or ''} " else: out += f"| {param.get(column, '')} " out += "|\n" # Top-level ungrouped parameters if len(self.schema.get("properties", {})) > 0: + required = self.schema.get("required", []) out += "\n## Other parameters\n\n" out += "".join([f"| {column.title()} " for column in columns]) out += "|\n" From 66e6f12663f6f38c7a3586f17d358949837daeb3 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 10 Jan 2023 15:37:19 +0000 Subject: [PATCH 811/854] Refactor param table generation from schema to be its own method --- nf_core/schema.py | 83 ++++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 1d36d45c12..59a47679fe 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -495,54 +495,57 @@ def schema_to_markdown(self, columns): out += f"{self.schema['description']}\n" # Grouped parameters for definition in self.schema.get("definitions", {}).values(): - required = definition.get("required", []) out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" - out += "".join([f"| {column.title()} " for column in columns]) - out += "|\n" - out += "".join(["|-----------" for columns in columns]) - out += "|\n" - for p_key, param in definition.get("properties", {}).items(): - for column in columns: - if column == "parameter": - out += f"| `{p_key}` " - elif column == "description": - desc = param.get("description", "").replace("\n", "
    ") - out += f"| {desc} " - if param.get("help_text", "") != "": - help_txt = param["help_text"].replace("\n", "
    ") - out += f"
    Help{help_txt}
    " - elif column == "type": - out += f"| `{param.get('type', '')}` " - elif column == "required": - out += f"| {p_key in required or ''} " - else: - out += f"| {param.get(column, '')} " - out += "|\n" + required = definition.get("required", []) + properties = definition.get("properties", {}).items() + param_table = self.markdown_param_table_content(properties, required, columns) + out += param_table # Top-level ungrouped parameters if len(self.schema.get("properties", {})) > 0: - required = self.schema.get("required", []) out += "\n## Other parameters\n\n" - out += "".join([f"| {column.title()} " for column in columns]) - out += "|\n" - out += "".join(["|-----------" for columns in columns]) - out += "|\n" + required = self.schema.get("required", []) + properties = self.schema.get("properties", {}) + param_table = self.markdown_param_table_content(properties, required, columns) + out += param_table + + return out + + def markdown_param_table(properties, required, columns): + """Creates a markdown table for params from jsonschema properties section - for p_key, param in self.schema.get("properties", {}).items(): - for column in columns: - if column == "parameter": - out += f"| `{p_key}` " - elif column == "description": - out += f"| {param.get('description', '')} " - if param.get("help_text", "") != "": - out += f"
    Help{param['help_text']}
    " - elif column == "type": - out += f"| `{param.get('type', '')}` " - else: - out += f"| {param.get(column, '')} " - out += "|\n" + Args: + properties (dict): A jsonschema properties dictionary + required (list): A list of the required fields. + Should come from the same level of the jsonschema as properties + columns (list): A list of columns to write + Returns: + str: A string with the markdown table + """ + out = "" + out += "".join([f"| {column.title()} " for column in columns]) + out += "|\n" + out += "".join(["|-----------" for _ in columns]) + out += "|\n" + for p_key, param in properties.items(): + for column in columns: + if column == "parameter": + out += f"| `{p_key}` " + elif column == "description": + desc = param.get("description", "").replace("\n", "
    ") + out += f"| {desc} " + if param.get("help_text", "") != "": + help_txt = param["help_text"].replace("\n", "
    ") + out += f"
    Help{help_txt}
    " + elif column == "type": + out += f"| `{param.get('type', '')}` " + elif column == "required": + out += f"| {p_key in required or ''} " + else: + out += f"| {param.get(column, '')} " + out += "|\n" return out def markdown_to_html(self, markdown_str): From b63a407bdb06800e5b62703dafcad4be57831875 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 10 Jan 2023 15:48:00 +0000 Subject: [PATCH 812/854] Fix markdown table method signature --- nf_core/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 59a47679fe..ba88e762ea 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -498,8 +498,8 @@ def schema_to_markdown(self, columns): out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" required = definition.get("required", []) - properties = definition.get("properties", {}).items() - param_table = self.markdown_param_table_content(properties, required, columns) + properties = definition.get("properties", {}) + param_table = self.markdown_param_table(properties, required, columns) out += param_table # Top-level ungrouped parameters @@ -507,12 +507,12 @@ def schema_to_markdown(self, columns): out += "\n## Other parameters\n\n" required = self.schema.get("required", []) properties = self.schema.get("properties", {}) - param_table = self.markdown_param_table_content(properties, required, columns) + param_table = self.markdown_param_table(properties, required, columns) out += param_table return out - def markdown_param_table(properties, required, columns): + def markdown_param_table(self, properties, required, columns): """Creates a markdown table for params from jsonschema properties section Args: From 70ab7827d46edfb0172a5d7b9066ad96aef07050 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 20 Jan 2023 13:00:30 +0100 Subject: [PATCH 813/854] Activate auto-work clean up on successful completion for full-tests --- nf_core/pipeline-template/conf/test_full.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index d92692fa94..46b165a910 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -10,6 +10,8 @@ ---------------------------------------------------------------------------------------- */ +cleanup = true + params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' From 67c8aa9d36493fef7b9b1e6075cf50dee4ab7c95 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 20 Jan 2023 13:02:34 +0100 Subject: [PATCH 814/854] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be6d5820d7..856bc9a604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) + ### Linting ### Modules From 2a4b23b909155adef85dde79adc341db97995d99 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 24 Jan 2023 09:54:05 +0100 Subject: [PATCH 815/854] fix use of component_type --- nf_core/components/remove.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 528f032124..99df757992 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -127,7 +127,7 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals component, silent=True ): log.warning( - f"Could not install the {self.component_type[:-1]} '{component}', please install it manually with 'nf-core {component_type} install {component}'." + f"Could not install the {self.component_type[:-1]} '{component}', please install it manually with 'nf-core {self.component_type} install {component}'." ) removed_components.append(component) return removed From 51f16643b2bcd793292df2bed50244e9f3485db1 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 24 Jan 2023 09:57:33 +0100 Subject: [PATCH 816/854] fix argument name in docstring --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cda7c827dc..28481a9c3f 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -494,7 +494,7 @@ def reinstall_repo(self, install_dir, remote_url, module_entries): Args: install_dir (str): The name of directory where modules are installed remote_url (str): The git url of the remote repository - modules ([ dict[str, dict[str, str]] ]): Module entries with + module_entries ([ dict[str, dict[str, str]] ]): Module entries with branch and git sha info Returns: From 5c86b52bac04c39f4c057c890780a2eda7ab788a Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 24 Jan 2023 09:59:52 +0100 Subject: [PATCH 817/854] differentiate variable name from builtin function --- nf_core/modules/modules_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 28481a9c3f..d82f206e3e 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -419,10 +419,10 @@ def unsynced_components(self): def parse_dirs(self, dirs, missing_installation, component_type): untracked_dirs = [] - for dir in dirs: + for dir_ in dirs: # Check if the module/subworkflows directory exists in modules.json - install_dir = dir.parts[0] - component = str(Path(*dir.parts[1:])) + install_dir = dir_.parts[0] + component = str(Path(*dir_.parts[1:])) component_in_file = False git_url = None for repo in missing_installation: From da6f860e83d6a32959aa9acc4924cf7c2522201b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 24 Jan 2023 19:32:57 +0100 Subject: [PATCH 818/854] Make jobs automatically resubmit for a much wider range of exit codes now `130..145` --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/base.config | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 856bc9a604..0e8e84504b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) +- Make jobs automatically resubmit for a much wider range of exit codes (now `130..145`) ### Linting diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index c5c691057d..59622840f2 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in 130..145 ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' From e5ce2ab833b5b9f160eb76bcb3dadb17951bb702 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:05:35 +0100 Subject: [PATCH 819/854] Update usage.md --- nf_core/pipeline-template/docs/usage.md | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 9a171f5aad..acab79dfb7 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -75,6 +75,29 @@ work # Directory containing the nextflow working files # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` +If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file. + +Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. + +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. +The above pipeline run specified with a params file in yaml format: + +```bash +nextflow run {{ name }} -profile docker -params-file params.yaml +``` + +with `params.yaml` containing: + +```yaml +input: './samplesheet.csv' +outdir: './results/' +genome: 'GRCh37' +input: 'data' +<...> +``` + +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). + ### Updating the pipeline When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: @@ -91,6 +114,10 @@ First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releas This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. +To further assist in reproducbility, you can use share and re-use [params files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. + +> 💡 If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. + ## Core Nextflow arguments > **NB:** These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). From f1d3f14e1a8c61481d40e3b24e7459b112e51698 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:06:50 +0100 Subject: [PATCH 820/854] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 856bc9a604..759375d4f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ### Template -- Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) +- Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] +- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133] ### Linting From aa9f6565c31e3f286368bae7e22f7a3add88517c Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:07:13 +0100 Subject: [PATCH 821/854] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 759375d4f1..ac57b68d32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] -- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133] +- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] ### Linting From 263a487844be29023ca084cd9513064a7a09c951 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 27 Jan 2023 08:16:48 +0000 Subject: [PATCH 822/854] [automated] Fix code linting --- CHANGELOG.md | 2 +- nf_core/pipeline-template/docs/usage.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac57b68d32..575049e505 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] -- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] +- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] ### Linting diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index acab79dfb7..5581680ae6 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -80,7 +80,7 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. > ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. -The above pipeline run specified with a params file in yaml format: +> The above pipeline run specified with a params file in yaml format: ```bash nextflow run {{ name }} -profile docker -params-file params.yaml @@ -96,7 +96,7 @@ input: 'data' <...> ``` -You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). ### Updating the pipeline From 5e16056a77c53e126494182d2422bf4966da34d9 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:40:02 +0100 Subject: [PATCH 823/854] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/docs/usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 5581680ae6..24ed0fb4bf 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -79,7 +79,7 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. -> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. > The above pipeline run specified with a params file in yaml format: ```bash @@ -114,7 +114,7 @@ First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releas This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. -To further assist in reproducbility, you can use share and re-use [params files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. +To further assist in reproducbility, you can use share and re-use [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. > 💡 If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. From 579a0c3c6d45ed1dc061c64bfacc1e38c900ce99 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 13:24:32 +0100 Subject: [PATCH 824/854] Update nf_core/pipeline-template/docs/usage.md Co-authored-by: Daniel Straub <42973691+d4straub@users.noreply.github.com> --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 24ed0fb4bf..3a87c2f9de 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -79,7 +79,7 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. -> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). > The above pipeline run specified with a params file in yaml format: ```bash From eeba92b6f4c9c774876a2cc19e00696fa09dfaeb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 30 Jan 2023 10:46:21 +0100 Subject: [PATCH 825/854] Add 104 to retry error codes --- nf_core/pipeline-template/conf/base.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 59622840f2..f73c5afaa4 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in 130..145 ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' From c0a6ff56bcf5a599dd86928f1652ae816dc54390 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 1 Feb 2023 01:33:58 +0100 Subject: [PATCH 826/854] Update CHANGELOG.md Co-authored-by: Matthieu Muffato --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e8e84504b..691a3a0b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) -- Make jobs automatically resubmit for a much wider range of exit codes (now `130..145`) +- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ### Linting From 9203455d4e31ceb2ff0ca55da9f875a10032bf62 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Feb 2023 16:14:01 +0100 Subject: [PATCH 827/854] modify jinja2 modules template --- nf_core/module-template/modules/main.nf | 18 +++++++++++++-- nf_core/module-template/modules/meta.yml | 28 +++++++++++++++++------- nf_core/module-template/tests/test.yml | 4 ++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index e8f043f083..c9fe8361b5 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,3 +1,4 @@ +{% if not_minimal -%} // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/modules/nf-core/ // You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: @@ -14,33 +15,42 @@ // bwa mem | samtools view -B -T ref.fasta // TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty // list (`[]`) instead of a file can be used to work around this issue. +{%- endif %} process {{ tool_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' + {% if not_minimal -%} // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. + {%- endif %} conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'quay.io/biocontainers/YOUR-TOOL-HERE' }}' }" input: + {% if not_minimal -%} // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" // MUST be provided as an input via a Groovy Map called "meta". // This information may not be required in some instances e.g. indexing reference genome files: // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. - {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} + {%- endif %} + {{ {'tuple val(meta), path(bam)' if not_minimal else 'tuple val(meta), path(input)'} if has_meta else {'path bam' if not_minimal else 'path input'} }} output: + {% if not_minimal -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels - {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam + {%- endif %} + {{ {'tuple val(meta), path("*.bam")' if not_minimal else 'tuple val(meta), path("*")'} if has_meta else {'path "*.bam"' if not_minimal else 'path "*"'} }}, emit: {bam if not_minimal else output} + {% if not_minimal -%} // TODO nf-core: List additional required output channels/values here + {%- endif %} path "versions.yml" , emit: versions when: @@ -51,6 +61,7 @@ process {{ tool_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} + {% if not_minimal -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -60,7 +71,9 @@ process {{ tool_name_underscore|upper }} { // using the Nextflow "task" variable e.g. "--threads $task.cpus" // TODO nf-core: Please replace the example samtools command below with your module's command // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + {%- endif %} """ + {% if not_minimal -%} samtools \\ sort \\ $args \\ @@ -70,6 +83,7 @@ process {{ tool_name_underscore|upper }} { -T $prefix \\ {%- endif %} $bam + {%- endif %} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index c95e3e1d84..fa3011585b 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,11 +1,15 @@ name: "{{ tool_name_underscore }}" +#{% if not_minimal -%} ## TODO nf-core: Add a description of the module and list keywords +#{%- endif %} description: write your description here keywords: - sort tools: - "{{ tool }}": + #{% if not_minimal -%} ## TODO nf-core: Add a description and other details for the software below + #{%- endif %} description: "{{ tool_description }}" homepage: "{{ tool_doc_url }}" documentation: "{{ tool_doc_url }}" @@ -13,7 +17,9 @@ tools: doi: "" licence: "{{ tool_licence }}" +#{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as input +#{%- endif %} input: #{% if has_meta %} Only when we have meta - meta: @@ -22,13 +28,17 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] #{% endif %} + #{% if not_minimal -%} ## TODO nf-core: Delete / customise this example input - - bam: - type: file - description: BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" + #{%- endif %} + - #{{ 'bam:' if not_minimal else "input:" }} + type: file + description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} +#{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as output +#{%- endif %} output: #{% if has_meta -%} Only when we have meta - meta: @@ -41,11 +51,13 @@ output: type: file description: File containing software versions pattern: "versions.yml" + #{% if not_minimal -%} ## TODO nf-core: Delete / customise this example output - - bam: - type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" + #{%- endif %} + - #{{ 'bam:' if not_minimal else "output:" }} + type: file + description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} authors: - "{{ author }}" diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index a65b127c97..4095c1108c 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,12 +1,16 @@ +#{% if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} +#{%- endif %} - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ tool_dir }}/nextflow.config tags: - "{{ tool }}{% if subtool -%}" - "{{ tool }}/{{ subtool }}{%- endif %}" files: + #{% if not_minimal -%} - path: "output/{{ tool }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ tool }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b + #{%- endif %} From f13cc5e8b1ac379cee2c9d35788499675ddc35c7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 09:31:50 +0100 Subject: [PATCH 828/854] run isort --- nf_core/components/create.py | 438 +++++++++++++++++++++++++++++++++ nf_core/modules/create.py | 232 +---------------- nf_core/subworkflows/create.py | 112 +-------- 3 files changed, 454 insertions(+), 328 deletions(-) create mode 100644 nf_core/components/create.py diff --git a/nf_core/components/create.py b/nf_core/components/create.py new file mode 100644 index 0000000000..d44eb02c47 --- /dev/null +++ b/nf_core/components/create.py @@ -0,0 +1,438 @@ +""" +The ComponentCreate class handles generating of module and subworkflow templates +""" + +from __future__ import print_function + +import glob +import json +import logging +import os +import re +import subprocess + +import jinja2 +import questionary +import rich +import yaml +from packaging.version import parse as parse_version + +import nf_core +import nf_core.components.components_create +import nf_core.utils +from nf_core.components.components_command import ComponentCommand + +log = logging.getLogger(__name__) + + +class ComponentCreate(ComponentCommand): + def __init__( + self, + component_type, + directory=".", + component="", + author=None, + process_label=None, + has_meta=None, + force=False, + conda_name=None, + conda_version=None, + ): + super().__init__(component_type, directory) + self.directory = directory + self.component = component + self.author = author + self.process_label = process_label + self.has_meta = has_meta + self.force_overwrite = force + self.subtool = None + self.tool_conda_name = conda_name + self.tool_conda_version = conda_version + self.tool_licence = None + self.tool_licence = "" + self.tool_description = "" + self.tool_doc_url = "" + self.tool_dev_url = "" + self.bioconda = None + self.singularity_container = None + self.docker_container = None + self.file_paths = {} + + def create(self): + """ + Create a new DSL2 module or subworkflow from the nf-core template. + + A module should be named just or + e.g fastqc or samtools/sort, respectively. + + The subworkflow should be named as the main file type it operates on and a short description of the task performed + e.g bam_sort or bam_sort_samtools, respectively. + + If is a pipeline, this function creates a file called: + '/modules/local/tool.nf' + OR + '/modules/local/tool_subtool.nf' + OR for subworkflows + '/subworkflows/local/subworkflow_name.nf' + + If is a clone of nf-core/modules, it creates or modifies the following files: + + For modules: + + modules/modules/nf-core/tool/subtool/ + * main.nf + * meta.yml + modules/tests/modules/nf-core/tool/subtool/ + * main.nf + * test.yml + * nextflow.config + tests/config/pytest_modules.yml + + The function will attempt to automatically find a Bioconda package called + and matching Docker / Singularity images from BioContainers. + + For subworkflows: + subworkflows/nf-core/subworkflow_name/ + * main.nf + * meta.yml + tests/subworkflows/nf-core/subworkflow_name/ + * main.nf + * test.yml + * nextflow.config + tests/config/pytest_modules.yml + + """ + + if self.component_type == "modules": + # Check modules directory structure + self.check_modules_structure() + + # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules + log.info(f"Repository type: [blue]{self.repo_type}") + if self.directory != ".": + log.info(f"Base directory: '{self.directory}'") + + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " + "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." + ) + + # Collect component info via prompt if empty or invalid + self.component, self.subtool = self._collect_name_prompt() + + # Determine the component name + self.component_name = self.component + self.component_dir = self.component + + if self.subtool: + self.component_name = f"{self.component}/{self.subtool}" + self.component_dir = os.path.join(self.component, self.subtool) + + self.component_name_underscore = self.component_name.replace("/", "_") + + # Check existence of directories early for fast-fail + self.file_paths = self._get_component_dirs() + + if self.component_type == "modules": + # Try to find a bioconda package for 'component' + self._get_bioconda_tool() + + # Prompt for GitHub username + self._get_username() + + self._get_module_structure_components() + + # Create component template with jinja2 + self._render_template() + + if self.repo_type == "modules": + # Add entry to pytest_modules.yml + try: + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: + pytest_modules_yml = yaml.safe_load(fh) + if self.subtool: + pytest_modules_yml[self.component_name] = [ + f"modules/{self.org}/{self.component}/{self.subtool}/**", + f"tests/modules/{self.org}/{self.component}/{self.subtool}/**", + ] + else: + pytest_modules_yml[ + ("" if self.component_type == "modules" else self.component_type + "/") + self.component_name + ] = [ + f"{self.component_type}/{self.org}/{self.component}/**", + f"tests/{self.component_type}/{self.org}/{self.component}/**", + ] + pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: + yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) + except FileNotFoundError: + raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") + + new_files = list(self.file_paths.values()) + if self.repo_type == "modules": + new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) + log.info("Created / edited following files:\n " + "\n ".join(new_files)) + + def _get_bioconda_tool(self): + """ + Try to find a bioconda package for 'tool' + """ + while True: + try: + if self.tool_conda_name: + anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) + else: + anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) + + if not self.tool_conda_version: + version = anaconda_response.get("latest_version") + if not version: + version = str(max([parse_version(v) for v in anaconda_response["versions"]])) + else: + version = self.tool_conda_version + + self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) + self.tool_description = anaconda_response.get("summary", "") + self.tool_doc_url = anaconda_response.get("doc_url", "") + self.tool_dev_url = anaconda_response.get("dev_url", "") + if self.tool_conda_name: + self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version + else: + self.bioconda = "bioconda::" + self.tool + "=" + version + log.info(f"Using Bioconda package: '{self.bioconda}'") + break + except (ValueError, LookupError) as e: + log.warning( + f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.tool}'" + ) + if rich.prompt.Confirm.ask("[violet]Do you want to enter a different Bioconda package name?"): + self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() + continue + else: + log.warning( + f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually." + ) + break + + # Try to get the container tag (only if bioconda package was found) + if self.bioconda: + try: + if self.tool_conda_name: + self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( + self.tool_conda_name, version + ) + else: + self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( + self.tool, version + ) + log.info(f"Using Docker container: '{self.docker_container}'") + log.info(f"Using Singularity container: '{self.singularity_container}'") + except (ValueError, LookupError) as e: + log.info(f"Could not find a Docker/Singularity container ({e})") + + def _get_module_structure_components(self): + process_label_defaults = ["process_single", "process_low", "process_medium", "process_high", "process_long"] + if self.process_label is None: + log.info( + "Provide an appropriate resource label for the process, taken from the " + "[link=https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config#L29]nf-core pipeline template[/link].\n" + "For example: {}".format(", ".join(process_label_defaults)) + ) + while self.process_label is None: + self.process_label = questionary.autocomplete( + "Process resource label:", + choices=process_label_defaults, + style=nf_core.utils.nfcore_question_style, + default="process_single", + ).unsafe_ask() + + if self.has_meta is None: + log.info( + "Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' " + "MUST be provided as an input via a Groovy Map called 'meta'. " + "This information may [italic]not[/] be required in some instances, for example " + "[link=https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf]indexing reference genome files[/link]." + ) + while self.has_meta is None: + self.has_meta = rich.prompt.Confirm.ask( + "[violet]Will the module require a meta map of sample information?", default=True + ) + + def _render_template(self): + """ + Create new module/subworkflow files with Jinja2. + """ + object_attrs = vars(self) + # Run jinja2 for each file in the template folder + env = jinja2.Environment( + loader=jinja2.PackageLoader("nf_core", f"{self.component_type[:-1]}-template"), keep_trailing_newline=True + ) + for template_fn, dest_fn in self.file_paths.items(): + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + object_attrs["nf_core_version"] = nf_core.__version__ + rendered_output = j_template.render(object_attrs) + + # Write output to the target file + os.makedirs(os.path.dirname(dest_fn), exist_ok=True) + with open(dest_fn, "w") as fh: + log.debug(f"Writing output to: '{dest_fn}'") + fh.write(rendered_output) + + # Mirror file permissions + template_stat = os.stat( + os.path.join(os.path.dirname(nf_core.__file__), f"{self.component_type[:-1]}-template", template_fn) + ) + os.chmod(dest_fn, template_stat.st_mode) + + def _collect_name_prompt(self): + """ + Collect module/subworkflow info via prompt if empty or invalid + """ + # Collect module info via prompt if empty or invalid + subname = None + if self.component_type == "modules": + pattern = r"[^a-z\d/]" + elif self.component_type == "subworkflows": + pattern = r"[^a-z\d_/]" + if self.component is None: + self.component = "" + while self.component == "" or re.search(pattern, self.component) or self.component.count("/") > 0: + # Check + auto-fix for invalid chacters + if re.search(pattern, self.component): + if self.component_type == "modules": + log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") + elif self.component_type == "subworkflows": + log.warning("Subworkflow name must be lower-case letters only, with no punctuation") + name_clean = re.sub(r"[^a-z\d/]", "", self.component.lower()) + if rich.prompt.Confirm.ask(f"[violet]Change '{self.component}' to '{name_clean}'?"): + self.component = name_clean + else: + self.component = "" + + if self.component_type == "modules": + # Split into tool and subtool + if self.component.count("/") > 1: + log.warning("Tool/subtool can have maximum one '/' character") + self.component = "" + elif self.component.count("/") == 1: + self.component, subname = self.component.split("/") + else: + subname = None # Reset edge case: entered '/subtool' as name and gone round loop again + + # Prompt for new entry if we reset + if self.component == "": + if self.component_type == "modules": + self.component = rich.prompt.Prompt.ask("[violet]Name of tool/subtool").strip() + elif self.component_type == "subworkflows": + self.component = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() + + if self.component_type == "modules": + return self.component, subname + elif self.component_type == "subworkflows": + return self.component + + def _get_component_dirs(self): + """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist + + Returns dict: keys are relative paths to template files, vals are target paths. + """ + file_paths = {} + if self.repo_type == "pipeline": + local_component_dir = os.path.join(self.directory, self.component_type, "local") + # Check whether component file already exists + component_file = os.path.join(local_component_dir, f"{self.component_name}.nf") + if os.path.exists(component_file) and not self.force_overwrite: + raise UserWarning( + f"{self.component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + if self.subtool and os.path.exists(os.path.join(local_component_dir, f"{self.component}.nf")): + raise UserWarning( + f"Module '{self.component}' exists already, cannot make subtool '{self.component_name}'" + ) + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{local_component_dir}/{self.component}_*.nf") + if not self.subtool and tool_glob: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" + ) + + # Set file paths + file_paths[os.path.join(self.component_type, "main.nf")] = component_file + + if self.repo_type == "modules": + software_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) + test_dir = os.path.join(self.directory, "tests", self.component_type, self.org, self.component_dir) + + # Check if module/subworkflow directories exist already + if os.path.exists(software_dir) and not self.force_overwrite: + raise UserWarning( + f"{self.component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite" + ) + if os.path.exists(test_dir) and not self.force_overwrite: + raise UserWarning( + f"{self.component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = os.path.join( + self.directory, self.component_type, self.org, self.component, "main.nf" + ) + parent_tool_test_nf = os.path.join( + self.directory, self.component_type, self.org, self.component, "main.nf" + ) + if self.subtool and os.path.exists(parent_tool_main_nf): + raise UserWarning( + f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" + ) + if self.subtool and os.path.exists(parent_tool_test_nf): + raise UserWarning( + f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{self.component_name}'" + ) + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob( + f"{os.path.join(self.directory, self.component_type, self.org, self.component)}/*/main.nf" + ) + if not self.subtool and tool_glob: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" + ) + + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(software_dir, "main.nf") + file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") + file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") + file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") + file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") + + return file_paths + + def _get_username(self): + """ + Prompt for GitHub username + """ + # Try to guess the current user if `gh` is installed + author_default = None + try: + with open(os.devnull, "w") as devnull: + gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) + author_default = f"@{gh_auth_user['login']}" + except Exception as e: + log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") + + # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex + github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") + while self.author is None or not github_username_regex.match(self.author): + if self.author is not None and not github_username_regex.match(self.author): + log.warning("Does not look like a valid GitHub username (must start with an '@')!") + self.author = rich.prompt.Prompt.ask( + f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", + default=author_default, + ) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 019a77c71f..bb744af021 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -1,30 +1,15 @@ -""" -The ModuleCreate class handles generating of module templates -""" - -from __future__ import print_function - import logging -import os -import questionary -import rich -import yaml -from packaging.version import parse as parse_version - -import nf_core -import nf_core.components.components_create -import nf_core.utils -from nf_core.components.components_command import ComponentCommand +from nf_core.components.create import ComponentCreate log = logging.getLogger(__name__) -class ModuleCreate(ComponentCommand): +class ModuleCreate(ComponentCreate): def __init__( self, - directory=".", - tool="", + pipeline_dir, + component="", author=None, process_label=None, has_meta=None, @@ -32,211 +17,6 @@ def __init__( conda_name=None, conda_version=None, ): - super().__init__("modules", directory) - self.directory = directory - self.tool = tool - self.author = author - self.process_label = process_label - self.has_meta = has_meta - self.force_overwrite = force - self.subtool = None - self.tool_conda_name = conda_name - self.tool_conda_version = conda_version - self.tool_licence = None - self.tool_licence = "" - self.tool_description = "" - self.tool_doc_url = "" - self.tool_dev_url = "" - self.bioconda = None - self.singularity_container = None - self.docker_container = None - self.file_paths = {} - - def create(self): - """ - Create a new DSL2 module from the nf-core template. - - Tool should be named just or - e.g fastqc or samtools/sort, respectively. - - If is a pipeline, this function creates a file called: - '/modules/local/tool.nf' - OR - '/modules/local/tool_subtool.nf' - - If is a clone of nf-core/modules, it creates or modifies the following files: - - modules/modules/nf-core/tool/subtool/ - * main.nf - * meta.yml - modules/tests/modules/nf-core/tool/subtool/ - * main.nf - * test.yml - * nextflow.config - tests/config/pytest_modules.yml - - The function will attempt to automatically find a Bioconda package called - and matching Docker / Singularity images from BioContainers. - """ - - # Check modules directory structure - self.check_modules_structure() - - log.info(f"Repository type: [blue]{self.repo_type}") - if self.directory != ".": - log.info(f"Base directory: '{self.directory}'") - - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " - "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." + super().__init__( + "modules", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version ) - - # Collect module info via prompt if empty or invalid - self.tool, self.subtool = nf_core.components.components_create.collect_name_prompt( - self.tool, self.component_type - ) - - # Determine the tool name - self.tool_name = self.tool - self.tool_dir = self.tool - - if self.subtool: - self.tool_name = f"{self.tool}/{self.subtool}" - self.tool_dir = os.path.join(self.tool, self.subtool) - - self.tool_name_underscore = self.tool_name.replace("/", "_") - - # Check existence of directories early for fast-fail - self.file_paths = nf_core.components.components_create.get_component_dirs( - self.component_type, - self.repo_type, - self.directory, - self.org, - self.tool_name, - self.tool, - self.subtool, - self.tool_dir, - self.force_overwrite, - ) - - # Try to find a bioconda package for 'tool' - self._get_bioconda_tool() - - # Prompt for GitHub username - self.author = nf_core.components.components_create.get_username(self.author) - - self._get_module_structure_components() - - # Create module template with jinja2 - nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) - - if self.repo_type == "modules": - # Add entry to pytest_modules.yml - try: - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: - pytest_modules_yml = yaml.safe_load(fh) - if self.subtool: - pytest_modules_yml[self.tool_name] = [ - f"modules/{self.org}/{self.tool}/{self.subtool}/**", - f"tests/modules/{self.org}/{self.tool}/{self.subtool}/**", - ] - else: - pytest_modules_yml[self.tool_name] = [ - f"modules/{self.org}/{self.tool}/**", - f"tests/modules/{self.org}/{self.tool}/**", - ] - pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: - yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError: - raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") - - new_files = list(self.file_paths.values()) - if self.repo_type == "modules": - new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) - log.info("Created / edited following files:\n " + "\n ".join(new_files)) - - def _get_bioconda_tool(self): - """ - Try to find a bioconda package for 'tool' - """ - while True: - try: - if self.tool_conda_name: - anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) - else: - anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) - - if not self.tool_conda_version: - version = anaconda_response.get("latest_version") - if not version: - version = str(max([parse_version(v) for v in anaconda_response["versions"]])) - else: - version = self.tool_conda_version - - self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) - self.tool_description = anaconda_response.get("summary", "") - self.tool_doc_url = anaconda_response.get("doc_url", "") - self.tool_dev_url = anaconda_response.get("dev_url", "") - if self.tool_conda_name: - self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version - else: - self.bioconda = "bioconda::" + self.tool + "=" + version - log.info(f"Using Bioconda package: '{self.bioconda}'") - break - except (ValueError, LookupError) as e: - log.warning( - f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.tool}'" - ) - if rich.prompt.Confirm.ask("[violet]Do you want to enter a different Bioconda package name?"): - self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() - continue - else: - log.warning( - f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually." - ) - break - - # Try to get the container tag (only if bioconda package was found) - if self.bioconda: - try: - if self.tool_conda_name: - self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.tool_conda_name, version - ) - else: - self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.tool, version - ) - log.info(f"Using Docker container: '{self.docker_container}'") - log.info(f"Using Singularity container: '{self.singularity_container}'") - except (ValueError, LookupError) as e: - log.info(f"Could not find a Docker/Singularity container ({e})") - - def _get_module_structure_components(self): - process_label_defaults = ["process_single", "process_low", "process_medium", "process_high", "process_long"] - if self.process_label is None: - log.info( - "Provide an appropriate resource label for the process, taken from the " - "[link=https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config#L29]nf-core pipeline template[/link].\n" - "For example: {}".format(", ".join(process_label_defaults)) - ) - while self.process_label is None: - self.process_label = questionary.autocomplete( - "Process resource label:", - choices=process_label_defaults, - style=nf_core.utils.nfcore_question_style, - default="process_single", - ).unsafe_ask() - - if self.has_meta is None: - log.info( - "Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' " - "MUST be provided as an input via a Groovy Map called 'meta'. " - "This information may [italic]not[/] be required in some instances, for example " - "[link=https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf]indexing reference genome files[/link]." - ) - while self.has_meta is None: - self.has_meta = rich.prompt.Confirm.ask( - "[violet]Will the module require a meta map of sample information?", default=True - ) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index e61f0c6c8d..bd4e5a4fbd 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -1,114 +1,22 @@ -""" -The SubworkflowCreate class handles generating of subworkflow templates -""" - -from __future__ import print_function - import logging -import os - -import yaml -import nf_core -import nf_core.components.components_create -import nf_core.utils -from nf_core.components.components_command import ComponentCommand +from nf_core.components.create import ComponentCreate log = logging.getLogger(__name__) -class SubworkflowCreate(ComponentCommand): +class SubworkflowCreate(ComponentCreate): def __init__( self, - directory=".", - subworkflow="", + pipeline_dir, + component="", author=None, + process_label=None, + has_meta=None, force=False, + conda_name=None, + conda_version=None, ): - super().__init__("subworkflows", directory) - self.directory = directory - self.subworkflow = subworkflow - self.author = author - self.force_overwrite = force - self.file_paths = {} - - def create(self): - """ - Create a new subworkflow from the nf-core template. - - The subworkflow should be named as the main file type it operates on and a short description of the task performed - e.g bam_sort or bam_sort_samtools, respectively. - - If is a pipeline, this function creates a file called: - '/subworkflows/local/subworkflow_name.nf' - - If is a clone of nf-core/modules, it creates or modifies the following files: - - subworkflows/nf-core/subworkflow_name/ - * main.nf - * meta.yml - tests/subworkflows/nf-core/subworkflow_name/ - * main.nf - * test.yml - * nextflow.config - tests/config/pytest_modules.yml - - """ - - # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules - log.info(f"Repository type: [blue]{self.repo_type}") - if self.directory != ".": - log.info(f"Base directory: '{self.directory}'") - - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " - "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." - ) - - # Collect module info via prompt if empty or invalid - self.subworkflow = nf_core.components.components_create.collect_name_prompt( - self.subworkflow, self.component_type - ) - - # Determine the tool name - self.subworkflow_name = self.subworkflow - self.subworkflow_dir = self.subworkflow - - # Check existence of directories early for fast-fail - self.file_paths = nf_core.components.components_create.get_component_dirs( - self.component_type, - self.repo_type, - self.directory, - self.org, - self.subworkflow_name, - None, - None, - self.subworkflow_dir, - self.force_overwrite, + super().__init__( + "subworkflows", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version ) - - # Prompt for GitHub username - self.author = nf_core.components.components_create.get_username(self.author) - - # Create subworkflow template with jinja2 - nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) - - if self.repo_type == "modules": - # Add entry to pytest_modules.yml - try: - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: - pytest_modules_yml = yaml.safe_load(fh) - pytest_modules_yml["subworkflows/" + self.subworkflow] = [ - f"subworkflows/{self.org}/{self.subworkflow}/**", - f"tests/subworkflows/{self.org}/{self.subworkflow}/**", - ] - pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: - yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError: - raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") - - new_files = list(self.file_paths.values()) - if self.repo_type == "modules": - new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) - log.info("Created / edited following files:\n " + "\n ".join(new_files)) From ccc3530237df9231d5244ca19c7b97537c2e7cb4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 09:32:25 +0100 Subject: [PATCH 829/854] remove unneeded common functions --- nf_core/components/components_create.py | 177 ------------------------ 1 file changed, 177 deletions(-) delete mode 100644 nf_core/components/components_create.py diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py deleted file mode 100644 index 86c42c0373..0000000000 --- a/nf_core/components/components_create.py +++ /dev/null @@ -1,177 +0,0 @@ -import glob -import json -import logging -import os -import re -import subprocess - -import jinja2 -import rich - -import nf_core.utils - -log = logging.getLogger(__name__) - - -def render_template(component_type, object_attrs, file_paths): - """ - Create new module/subworkflow files with Jinja2. - """ - # Run jinja2 for each file in the template folder - env = jinja2.Environment( - loader=jinja2.PackageLoader("nf_core", f"{component_type[:-1]}-template"), keep_trailing_newline=True - ) - for template_fn, dest_fn in file_paths.items(): - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - object_attrs["nf_core_version"] = nf_core.__version__ - rendered_output = j_template.render(object_attrs) - - # Write output to the target file - os.makedirs(os.path.dirname(dest_fn), exist_ok=True) - with open(dest_fn, "w") as fh: - log.debug(f"Writing output to: '{dest_fn}'") - fh.write(rendered_output) - - # Mirror file permissions - template_stat = os.stat( - os.path.join(os.path.dirname(nf_core.__file__), f"{component_type[:-1]}-template", template_fn) - ) - os.chmod(dest_fn, template_stat.st_mode) - - -def collect_name_prompt(name, component_type): - """ - Collect module/subworkflow info via prompt if empty or invalid - """ - # Collect module info via prompt if empty or invalid - subname = None - if component_type == "modules": - pattern = r"[^a-z\d/]" - elif component_type == "subworkflows": - pattern = r"[^a-z\d_/]" - if name is None: - name = "" - while name == "" or re.search(pattern, name) or name.count("/") > 0: - # Check + auto-fix for invalid chacters - if re.search(pattern, name): - if component_type == "modules": - log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") - elif component_type == "subworkflows": - log.warning("Subworkflow name must be lower-case letters only, with no punctuation") - name_clean = re.sub(r"[^a-z\d/]", "", name.lower()) - if rich.prompt.Confirm.ask(f"[violet]Change '{name}' to '{name_clean}'?"): - name = name_clean - else: - name = "" - - if component_type == "modules": - # Split into tool and subtool - if name.count("/") > 1: - log.warning("Tool/subtool can have maximum one '/' character") - name = "" - elif name.count("/") == 1: - name, subname = name.split("/") - else: - subname = None # Reset edge case: entered '/subtool' as name and gone round loop again - - # Prompt for new entry if we reset - if name == "": - if component_type == "modules": - name = rich.prompt.Prompt.ask("[violet]Name of tool/subtool").strip() - elif component_type == "subworkflows": - name = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() - - if component_type == "modules": - return name, subname - elif component_type == "subworkflows": - return name - - -def get_component_dirs(component_type, repo_type, directory, org, name, supername, subname, new_dir, force_overwrite): - """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist - - Returns dict: keys are relative paths to template files, vals are target paths. - """ - file_paths = {} - if repo_type == "pipeline": - local_component_dir = os.path.join(directory, component_type, "local") - # Check whether component file already exists - component_file = os.path.join(local_component_dir, f"{name}.nf") - if os.path.exists(component_file) and not force_overwrite: - raise UserWarning( - f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" - ) - - if component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - if subname and os.path.exists(os.path.join(local_component_dir, f"{supername}.nf")): - raise UserWarning(f"Module '{supername}' exists already, cannot make subtool '{name}'") - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_component_dir}/{supername}_*.nf") - if not subname and tool_glob: - raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'") - - # Set file paths - file_paths[os.path.join(component_type, "main.nf")] = component_file - - if repo_type == "modules": - software_dir = os.path.join(directory, component_type, org, new_dir) - test_dir = os.path.join(directory, "tests", component_type, org, new_dir) - - # Check if module/subworkflow directories exist already - if os.path.exists(software_dir) and not force_overwrite: - raise UserWarning(f"{component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite") - if os.path.exists(test_dir) and not force_overwrite: - raise UserWarning(f"{component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite") - - if component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = os.path.join(directory, component_type, org, supername, "main.nf") - parent_tool_test_nf = os.path.join(directory, component_type, org, supername, "main.nf") - if subname and os.path.exists(parent_tool_main_nf): - raise UserWarning(f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{name}'") - if subname and os.path.exists(parent_tool_test_nf): - raise UserWarning(f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{name}'") - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{os.path.join(directory, component_type, org, supername)}/*/main.nf") - if not subname and tool_glob: - raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'") - - # Set file paths - # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join(component_type, "main.nf")] = os.path.join(software_dir, "main.nf") - file_paths[os.path.join(component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") - - return file_paths - - -def get_username(author): - """ - Prompt for GitHub username - """ - # Try to guess the current user if `gh` is installed - author_default = None - try: - with open(os.devnull, "w") as devnull: - gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) - author_default = f"@{gh_auth_user['login']}" - except Exception as e: - log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") - - # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex - github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") - while author is None or not github_username_regex.match(author): - if author is not None and not github_username_regex.match(author): - log.warning("Does not look like a valid GitHub username (must start with an '@')!") - author = rich.prompt.Prompt.ask( - f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", - default=author_default, - ) - - return author From 190f5e3089644e2dcdc291d6c5287aa0a9797112 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 10:15:27 +0100 Subject: [PATCH 830/854] linting --- nf_core/__main__.py | 10 ++++++---- nf_core/components/create.py | 24 ++++++++++-------------- nf_core/module-template/modules/main.nf | 8 ++++++-- nf_core/modules/create.py | 12 +++++++++++- nf_core/subworkflows/create.py | 12 +++++++++++- 5 files changed, 44 insertions(+), 22 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 555783d8d4..d8911f9c98 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -678,7 +678,8 @@ def remove(ctx, dir, tool): @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") @click.option("-p", "--conda-package-version", type=str, default=None, help="Version of conda package to use") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version): +@click.option("-i", "--minimal", is_flag=True, default=False, help="Create a minimal version of the template") +def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, minimal): """ Create a new DSL2 module from the nf-core template. @@ -700,7 +701,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: module_create = nf_core.modules.ModuleCreate( - dir, tool, author, label, has_meta, force, conda_name, conda_package_version + dir, tool, author, label, has_meta, force, conda_name, conda_package_version, minimal ) module_create.create() except UserWarning as e: @@ -936,7 +937,8 @@ def test_module(ctx, tool, no_prompts, pytest_args): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") -def create_subworkflow(ctx, subworkflow, dir, author, force): +@click.option("-i", "--minimap", is_flag=True, default=False, help="Create a minimal version of the template") +def create_subworkflow(ctx, subworkflow, dir, author, force, minimal): """ Create a new subworkflow from the nf-core template. @@ -949,7 +951,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): # Run function try: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force, minimal) subworkflow_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index d44eb02c47..527cb0c556 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -18,7 +18,6 @@ from packaging.version import parse as parse_version import nf_core -import nf_core.components.components_create import nf_core.utils from nf_core.components.components_command import ComponentCommand @@ -37,6 +36,7 @@ def __init__( force=False, conda_name=None, conda_version=None, + minimal=False, ): super().__init__(component_type, directory) self.directory = directory @@ -57,6 +57,7 @@ def __init__( self.singularity_container = None self.docker_container = None self.file_paths = {} + self.not_minimal = not minimal def create(self): """ @@ -118,7 +119,7 @@ def create(self): ) # Collect component info via prompt if empty or invalid - self.component, self.subtool = self._collect_name_prompt() + self._collect_name_prompt() # Determine the component name self.component_name = self.component @@ -182,7 +183,7 @@ def _get_bioconda_tool(self): if self.tool_conda_name: anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) else: - anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) + anaconda_response = nf_core.utils.anaconda_package(self.component, ["bioconda"]) if not self.tool_conda_version: version = anaconda_response.get("latest_version") @@ -198,12 +199,12 @@ def _get_bioconda_tool(self): if self.tool_conda_name: self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version else: - self.bioconda = "bioconda::" + self.tool + "=" + version + self.bioconda = "bioconda::" + self.component + "=" + version log.info(f"Using Bioconda package: '{self.bioconda}'") break except (ValueError, LookupError) as e: log.warning( - f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.tool}'" + f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.component}'" ) if rich.prompt.Confirm.ask("[violet]Do you want to enter a different Bioconda package name?"): self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() @@ -223,7 +224,7 @@ def _get_bioconda_tool(self): ) else: self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.tool, version + self.component, version ) log.info(f"Using Docker container: '{self.docker_container}'") log.info(f"Using Singularity container: '{self.singularity_container}'") @@ -290,7 +291,7 @@ def _collect_name_prompt(self): Collect module/subworkflow info via prompt if empty or invalid """ # Collect module info via prompt if empty or invalid - subname = None + self.subtool = None if self.component_type == "modules": pattern = r"[^a-z\d/]" elif self.component_type == "subworkflows": @@ -316,9 +317,9 @@ def _collect_name_prompt(self): log.warning("Tool/subtool can have maximum one '/' character") self.component = "" elif self.component.count("/") == 1: - self.component, subname = self.component.split("/") + self.component, self.subtool = self.component.split("/") else: - subname = None # Reset edge case: entered '/subtool' as name and gone round loop again + self.subtool = None # Reset edge case: entered '/subtool' as name and gone round loop again # Prompt for new entry if we reset if self.component == "": @@ -327,11 +328,6 @@ def _collect_name_prompt(self): elif self.component_type == "subworkflows": self.component = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() - if self.component_type == "modules": - return self.component, subname - elif self.component_type == "subworkflows": - return self.component - def _get_component_dirs(self): """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index c9fe8361b5..3199e46367 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -40,14 +40,18 @@ process {{ tool_name_underscore|upper }} { // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. + {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} + {%- else %} + {{ 'tuple val(meta), path(input)' if has_meta else 'path input' }} {%- endif %} - {{ {'tuple val(meta), path(bam)' if not_minimal else 'tuple val(meta), path(input)'} if has_meta else {'path bam' if not_minimal else 'path input'} }} output: {% if not_minimal -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels + {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam + {%- else %} + {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {{ {'tuple val(meta), path("*.bam")' if not_minimal else 'tuple val(meta), path("*")'} if has_meta else {'path "*.bam"' if not_minimal else 'path "*"'} }}, emit: {bam if not_minimal else output} {% if not_minimal -%} // TODO nf-core: List additional required output channels/values here {%- endif %} diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index bb744af021..16d14ef6aa 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -16,7 +16,17 @@ def __init__( force=False, conda_name=None, conda_version=None, + minimal=False, ): super().__init__( - "modules", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version + "modules", + pipeline_dir, + component, + author, + process_label, + has_meta, + force, + conda_name, + conda_version, + minimal, ) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index bd4e5a4fbd..47181e477d 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -16,7 +16,17 @@ def __init__( force=False, conda_name=None, conda_version=None, + minimal=False, ): super().__init__( - "subworkflows", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version + "subworkflows", + pipeline_dir, + component, + author, + process_label, + has_meta, + force, + conda_name, + conda_version, + minimal, ) From f20614dfa4e425583e34b06ed452146f234b0380 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 10:53:58 +0100 Subject: [PATCH 831/854] refine new lines --- .prettierignore | 3 +- nf_core/module-template/modules/main.nf | 14 ++++---- nf_core/module-template/modules/meta.yml | 42 ++++++++++++------------ nf_core/module-template/tests/main.nf | 6 ++-- nf_core/module-template/tests/test.yml | 10 +++--- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.prettierignore b/.prettierignore index bd1a8bee9c..4cd77bb4ed 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,4 +3,5 @@ adaptivecard.json slackreport.json docs/api/_build testing - +nf_core/module-template/modules/meta.yml +nf_core/module-template/tests/test.yml diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 3199e46367..3736c91655 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -17,7 +17,7 @@ // list (`[]`) instead of a file can be used to work around this issue. {%- endif %} -process {{ tool_name_underscore|upper }} { +process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' @@ -26,7 +26,7 @@ process {{ tool_name_underscore|upper }} { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - {%- endif %} + {%- endif -%} conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': @@ -41,7 +41,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} - {%- else %} + {%- else -%} {{ 'tuple val(meta), path(input)' if has_meta else 'path input' }} {%- endif %} @@ -49,10 +49,10 @@ process {{ tool_name_underscore|upper }} { {% if not_minimal -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam - {%- else %} + {%- else -%} {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {% if not_minimal -%} + {%- if not_minimal -%} // TODO nf-core: List additional required output channels/values here {%- endif %} path "versions.yml" , emit: versions @@ -65,7 +65,7 @@ process {{ tool_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} - {% if not_minimal -%} + {%- if not_minimal -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -77,7 +77,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) {%- endif %} """ - {% if not_minimal -%} + {%- if not_minimal -%} samtools \\ sort \\ $args \\ diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index fa3011585b..6a949772ad 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,15 +1,15 @@ -name: "{{ tool_name_underscore }}" -#{% if not_minimal -%} +name: "{{ component_name_underscore }}" +{% if not_minimal -%} ## TODO nf-core: Add a description of the module and list keywords -#{%- endif %} +{%- endif -%} description: write your description here keywords: - sort tools: - "{{ tool }}": - #{% if not_minimal -%} + {% if not_minimal -%} ## TODO nf-core: Add a description and other details for the software below - #{%- endif %} + {%- endif -%} description: "{{ tool_description }}" homepage: "{{ tool_doc_url }}" documentation: "{{ tool_doc_url }}" @@ -17,9 +17,9 @@ tools: doi: "" licence: "{{ tool_licence }}" -#{% if not_minimal -%} +{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as input -#{%- endif %} +{%- endif -%} input: #{% if has_meta %} Only when we have meta - meta: @@ -27,18 +27,18 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - #{% endif %} - #{% if not_minimal -%} + {% endif %} + {% if not_minimal -%} ## TODO nf-core: Delete / customise this example input - #{%- endif %} - - #{{ 'bam:' if not_minimal else "input:" }} + {%- endif %} + - {{ 'bam:' if not_minimal else "input:" }} type: file - description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} -#{% if not_minimal -%} +{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as output -#{%- endif %} +{%- endif -%} output: #{% if has_meta -%} Only when we have meta - meta: @@ -46,18 +46,18 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - #{% endif %} + {% endif %} - versions: type: file description: File containing software versions pattern: "versions.yml" - #{% if not_minimal -%} + {% if not_minimal -%} ## TODO nf-core: Delete / customise this example output - #{%- endif %} - - #{{ 'bam:' if not_minimal else "output:" }} + {%- endif %} + - {{ 'bam:' if not_minimal else "output:" }} type: file - description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} authors: - "{{ author }}" diff --git a/nf_core/module-template/tests/main.nf b/nf_core/module-template/tests/main.nf index 351de72385..fcb7195fe4 100644 --- a/nf_core/module-template/tests/main.nf +++ b/nf_core/module-template/tests/main.nf @@ -2,9 +2,9 @@ nextflow.enable.dsl = 2 -include { {{ tool_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/{{ org }}/{{ tool_dir }}/main.nf' +include { {{ component_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/{{ org }}/{{ component_dir }}/main.nf' -workflow test_{{ tool_name_underscore }} { +workflow test_{{ component_name_underscore }} { {% if has_meta %} input = [ [ id:'test', single_end:false ], // meta map @@ -14,5 +14,5 @@ workflow test_{{ tool_name_underscore }} { input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) {%- endif %} - {{ tool_name_underscore|upper }} ( input ) + {{ component_name_underscore|upper }} ( input ) } diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 4095c1108c..5483f3f70f 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,16 +1,16 @@ -#{% if not_minimal -%} +{%- if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -#{%- endif %} +{%- endif %} - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" - command: nextflow run ./tests/modules/{{ org }}/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ tool_dir }}/nextflow.config + command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: - "{{ tool }}{% if subtool -%}" - "{{ tool }}/{{ subtool }}{%- endif %}" files: - #{% if not_minimal -%} + {%- if not_minimal -%} - path: "output/{{ tool }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ tool }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b - #{%- endif %} + {%- endif %} From cb8c6122a2f012b4e816a3ef5d1fdb82866c16eb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 12:36:54 +0100 Subject: [PATCH 832/854] run isort --- nf_core/__main__.py | 5 ++--- nf_core/components/create.py | 3 ++- nf_core/module-template/modules/meta.yml | 8 ++++---- nf_core/module-template/tests/test.yml | 14 ++++++++------ nf_core/subworkflows/create.py | 12 +----------- tests/subworkflows/create.py | 2 +- 6 files changed, 18 insertions(+), 26 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d8911f9c98..b02348c753 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -937,8 +937,7 @@ def test_module(ctx, tool, no_prompts, pytest_args): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") -@click.option("-i", "--minimap", is_flag=True, default=False, help="Create a minimal version of the template") -def create_subworkflow(ctx, subworkflow, dir, author, force, minimal): +def create_subworkflow(ctx, subworkflow, dir, author, force): """ Create a new subworkflow from the nf-core template. @@ -951,7 +950,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, minimal): # Run function try: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force, minimal) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force) subworkflow_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 527cb0c556..fc31aac306 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -141,7 +141,8 @@ def create(self): # Prompt for GitHub username self._get_username() - self._get_module_structure_components() + if self.component_type == "modules": + self._get_module_structure_components() # Create component template with jinja2 self._render_template() diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 6a949772ad..236fab2aec 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -6,10 +6,10 @@ description: write your description here keywords: - sort tools: - - "{{ tool }}": + - "{{ component }}": {% if not_minimal -%} ## TODO nf-core: Add a description and other details for the software below - {%- endif -%} + {% endif -%} description: "{{ tool_description }}" homepage: "{{ tool_doc_url }}" documentation: "{{ tool_doc_url }}" @@ -19,7 +19,7 @@ tools: {% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as input -{%- endif -%} +{% endif -%} input: #{% if has_meta %} Only when we have meta - meta: @@ -38,7 +38,7 @@ input: {% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as output -{%- endif -%} +{% endif -%} output: #{% if has_meta -%} Only when we have meta - meta: diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 5483f3f70f..deabcf23a9 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -2,15 +2,17 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} {%- endif %} -- name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" +- name: "{{ component }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: - - "{{ tool }}{% if subtool -%}" - - "{{ tool }}/{{ subtool }}{%- endif %}" + - "{{ component }}{% if subtool -%}" + - "{{ component }}/{{ subtool }}{%- endif %}" files: - {%- if not_minimal -%} - - path: "output/{{ tool }}/test.bam" + {% if not_minimal -%} + - path: "output/{{ component }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: "output/{{ tool }}/versions.yml" + - path: "output/{{ component }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b + {% else %} + - path: "" {%- endif %} diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 47181e477d..963076455e 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -11,22 +11,12 @@ def __init__( pipeline_dir, component="", author=None, - process_label=None, - has_meta=None, force=False, - conda_name=None, - conda_version=None, - minimal=False, ): super().__init__( "subworkflows", pipeline_dir, component, author, - process_label, - has_meta, - force, - conda_name, - conda_version, - minimal, + force=force, ) diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index eac4929136..60ee6add9a 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -28,7 +28,7 @@ def test_subworkflows_create_fail_exists(self): def test_subworkflows_create_nfcore_modules(self): """Create a subworkflow in nf-core/modules clone""" subworkflow_create = nf_core.subworkflows.SubworkflowCreate( - self.nfcore_modules, "test_subworkflow", "@author", True + self.nfcore_modules, "test_subworkflow", "@author", force=True ) subworkflow_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf")) From c65b36c10d22588e65d9223b3d67432af7270cd9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 13:13:12 +0100 Subject: [PATCH 833/854] fix some more new lines --- nf_core/module-template/modules/main.nf | 4 ++-- nf_core/module-template/tests/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 3736c91655..6bfa9eaa92 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,4 +1,4 @@ -{% if not_minimal -%} +{%- if not_minimal -%} // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/modules/nf-core/ // You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: @@ -15,7 +15,7 @@ // bwa mem | samtools view -B -T ref.fasta // TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty // list (`[]`) instead of a file can be used to work around this issue. -{%- endif %} +{%- endif -%} process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index deabcf23a9..61486b9249 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,7 +1,7 @@ {%- if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -{%- endif %} +{%- endif -%} - name: "{{ component }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: @@ -13,6 +13,6 @@ md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ component }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b - {% else %} + {% else -%} - path: "" {%- endif %} From a54b54a9673a2db9a2b7732dcac7bc9960a0c95e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 13:20:43 +0100 Subject: [PATCH 834/854] run black with new version --- nf_core/bump_version.py | 2 -- nf_core/create.py | 1 - nf_core/download.py | 1 - nf_core/launch.py | 3 --- nf_core/lint/__init__.py | 1 - nf_core/lint/files_unchanged.py | 2 -- nf_core/lint/template_strings.py | 1 - nf_core/list.py | 1 - nf_core/modules/bump_versions.py | 1 - nf_core/utils.py | 2 -- tests/test_download.py | 1 - tests/test_test_utils.py | 1 - 12 files changed, 17 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 1f38775bb6..129016fa38 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -133,12 +133,10 @@ def update_file_version(filename, pipeline_obj, patterns): replacements = [] for pattern in patterns: - found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/create.py b/nf_core/create.py index 045c35d1b4..74c9df1b87 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -283,7 +283,6 @@ def render_template(self): # Set the paths to skip according to customization for template_fn_path_obj in template_files: - template_fn_path = str(template_fn_path_obj) # Skip files that are in the self.skip_paths list diff --git a/nf_core/download.py b/nf_core/download.py index 2f964b3afd..9d430e3352 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -506,7 +506,6 @@ def get_singularity_images(self): containers_download = [] containers_pull = [] for container in self.containers: - # Fetch the output and cached filenames for this container out_path, cache_path = self.singularity_image_filenames(container) diff --git a/nf_core/launch.py b/nf_core/launch.py index 87150172f7..648c8775f8 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -98,7 +98,6 @@ def __init__( self.cli_launch = True def launch_pipeline(self): - # Prompt for pipeline if not supplied and no web launch ID if self.pipeline is None and self.web_id is None: launch_type = questionary.select( @@ -461,7 +460,6 @@ def prompt_group(self, group_id, group_obj): answers = {} error_msgs = [] while not while_break: - if len(error_msgs) == 0: self.print_param_header(group_id, group_obj, True) @@ -698,7 +696,6 @@ def build_command(self): # Pipeline parameters if len(self.schema_obj.input_params) > 0: - # Write the user selection to a file and run nextflow with that if self.use_params_file: dump_json_with_prettier(self.params_out, self.schema_obj.input_params) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index b46399eb97..e014a933ea 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -434,7 +434,6 @@ def format_result(test_results): ) def _print_summary(self): - # Summary table summary_colour = "red" if len(self.failed) > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index cadced5483..c0be64d0d7 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -147,7 +147,6 @@ def _tf(file_path): # Files that must be completely unchanged from template for files in files_exact: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): @@ -177,7 +176,6 @@ def _tf(file_path): # Files that can be added to, but that must contain the template contents for files in files_partial: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index 436abe7b2b..fb1f0f32e5 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -24,7 +24,6 @@ def template_strings(self): # Loop through files, searching for string num_matches = 0 for fn in self.files: - # Skip binary files binary_ftypes = ["image", "application/java-archive"] (ftype, encoding) = mimetypes.guess_type(fn) diff --git a/nf_core/list.py b/nf_core/list.py index 53307ac9bd..77a9ac3919 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -328,7 +328,6 @@ def get_local_nf_workflow_details(self): """Get full details about a local cached workflow""" if self.local_path is None: - # Try to guess the local cache directory if len(os.environ.get("NXF_ASSETS", "")) > 0: nf_wfdir = os.path.join(os.environ.get("NXF_ASSETS"), self.full_name) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 7b9702622e..6d61d4c750 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -187,7 +187,6 @@ def bump_module_version(self, module): found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/utils.py b/nf_core/utils.py index b60f61fff0..55703ab450 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -872,7 +872,6 @@ def get_repo_releases_branches(pipeline, wfs): # Repo is a nf-core pipeline for wf in wfs.remote_workflows: if wf.full_name == pipeline or wf.name == pipeline: - # Set to full name just in case it didn't have the nf-core/ prefix pipeline = wf.full_name @@ -883,7 +882,6 @@ def get_repo_releases_branches(pipeline, wfs): # Arbitrary GitHub repo else: if pipeline.count("/") == 1: - # Looks like a GitHub address - try working with this repo log.debug( f"Pipeline '{pipeline}' not in nf-core, but looks like a GitHub address - fetching releases from API" diff --git a/tests/test_download.py b/tests/test_download.py index 4577a83992..e2ae882394 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -18,7 +18,6 @@ class DownloadTest(unittest.TestCase): - # # Tests for 'get_release_hash' # diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index ddf88ef74a..c4e3d49ae0 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -33,7 +33,6 @@ def test_tmp_folder_does_not_exist_after(): def test_set_wd(): - with tempfile.TemporaryDirectory() as tmpdirname: with set_wd(tmpdirname): context_wd = Path().resolve() From c89a94c6eb0f6f861c966c35331e450216234cf3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 13:56:54 +0100 Subject: [PATCH 835/854] make tests pass because of template new lines --- nf_core/module-template/modules/main.nf | 10 +++++----- nf_core/module-template/tests/test.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 6bfa9eaa92..d7a2bb9826 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -15,7 +15,7 @@ // bwa mem | samtools view -B -T ref.fasta // TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty // list (`[]`) instead of a file can be used to work around this issue. -{%- endif -%} +{%- endif %} process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} @@ -26,7 +26,7 @@ process {{ component_name_underscore|upper }} { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - {%- endif -%} + {% endif -%} conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': @@ -52,7 +52,7 @@ process {{ component_name_underscore|upper }} { {%- else -%} {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {%- if not_minimal -%} + {% if not_minimal -%} // TODO nf-core: List additional required output channels/values here {%- endif %} path "versions.yml" , emit: versions @@ -65,7 +65,7 @@ process {{ component_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} - {%- if not_minimal -%} + {% if not_minimal -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -77,7 +77,7 @@ process {{ component_name_underscore|upper }} { // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) {%- endif %} """ - {%- if not_minimal -%} + {% if not_minimal -%} samtools \\ sort \\ $args \\ diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 61486b9249..21fa60dd5e 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,7 +1,7 @@ {%- if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -{%- endif -%} +{% endif -%} - name: "{{ component }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: From 5c5ae91c71fc24dd9ef9a714e16f2fdb40f04cd7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 14:45:26 +0100 Subject: [PATCH 836/854] update CHANGELOG --- CHANGELOG.md | 2 ++ nf_core/__main__.py | 14 +++++++++++--- nf_core/components/create.py | 4 ++-- nf_core/module-template/modules/main.nf | 14 +++++++------- nf_core/module-template/modules/meta.yml | 24 ++++++++++++------------ nf_core/module-template/tests/test.yml | 4 ++-- nf_core/modules/create.py | 4 ++-- 7 files changed, 38 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 575049e505..0c86f897f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ ### Modules +- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175)) + ### Subworkflows ### General diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b02348c753..1372ff7fbc 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -678,8 +678,16 @@ def remove(ctx, dir, tool): @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") @click.option("-p", "--conda-package-version", type=str, default=None, help="Version of conda package to use") -@click.option("-i", "--minimal", is_flag=True, default=False, help="Create a minimal version of the template") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, minimal): +@click.option( + "-i", + "--empty-template", + is_flag=True, + default=False, + help="Create a version of the template without TODOs or examples", +) +def create_module( + ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template +): """ Create a new DSL2 module from the nf-core template. @@ -701,7 +709,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: module_create = nf_core.modules.ModuleCreate( - dir, tool, author, label, has_meta, force, conda_name, conda_package_version, minimal + dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template ) module_create.create() except UserWarning as e: diff --git a/nf_core/components/create.py b/nf_core/components/create.py index fc31aac306..e626de4aaa 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -36,7 +36,7 @@ def __init__( force=False, conda_name=None, conda_version=None, - minimal=False, + empty_template=False, ): super().__init__(component_type, directory) self.directory = directory @@ -57,7 +57,7 @@ def __init__( self.singularity_container = None self.docker_container = None self.file_paths = {} - self.not_minimal = not minimal + self.not_empty_template = not empty_template def create(self): """ diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index d7a2bb9826..895ad8f68c 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,4 +1,4 @@ -{%- if not_minimal -%} +{%- if not_empty_template -%} // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/modules/nf-core/ // You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: @@ -21,7 +21,7 @@ process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. @@ -33,7 +33,7 @@ process {{ component_name_underscore|upper }} { '{{ docker_container if docker_container else 'quay.io/biocontainers/YOUR-TOOL-HERE' }}' }" input: - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" // MUST be provided as an input via a Groovy Map called "meta". // This information may not be required in some instances e.g. indexing reference genome files: @@ -46,13 +46,13 @@ process {{ component_name_underscore|upper }} { {%- endif %} output: - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam {%- else -%} {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: List additional required output channels/values here {%- endif %} path "versions.yml" , emit: versions @@ -65,7 +65,7 @@ process {{ component_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -77,7 +77,7 @@ process {{ component_name_underscore|upper }} { // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) {%- endif %} """ - {% if not_minimal -%} + {% if not_empty_template -%} samtools \\ sort \\ $args \\ diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 236fab2aec..05ecb6f980 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,5 +1,5 @@ name: "{{ component_name_underscore }}" -{% if not_minimal -%} +{% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords {%- endif -%} description: write your description here @@ -7,7 +7,7 @@ keywords: - sort tools: - "{{ component }}": - {% if not_minimal -%} + {% if not_empty_template -%} ## TODO nf-core: Add a description and other details for the software below {% endif -%} description: "{{ tool_description }}" @@ -17,7 +17,7 @@ tools: doi: "" licence: "{{ tool_licence }}" -{% if not_minimal -%} +{% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as input {% endif -%} input: @@ -28,15 +28,15 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] {% endif %} - {% if not_minimal -%} + {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example input {%- endif %} - - {{ 'bam:' if not_minimal else "input:" }} + - {{ 'bam:' if not_empty_template else "input:" }} type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} -{% if not_minimal -%} +{% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as output {% endif -%} output: @@ -51,13 +51,13 @@ output: type: file description: File containing software versions pattern: "versions.yml" - {% if not_minimal -%} + {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example output {%- endif %} - - {{ 'bam:' if not_minimal else "output:" }} + - {{ 'bam:' if not_empty_template else "output:" }} type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} authors: - "{{ author }}" diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 21fa60dd5e..4f38dec298 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,4 +1,4 @@ -{%- if not_minimal -%} +{%- if not_empty_template -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} {% endif -%} @@ -8,7 +8,7 @@ - "{{ component }}{% if subtool -%}" - "{{ component }}/{{ subtool }}{%- endif %}" files: - {% if not_minimal -%} + {% if not_empty_template -%} - path: "output/{{ component }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ component }}/versions.yml" diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 16d14ef6aa..b5368130ce 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -16,7 +16,7 @@ def __init__( force=False, conda_name=None, conda_version=None, - minimal=False, + empty_template=False, ): super().__init__( "modules", @@ -28,5 +28,5 @@ def __init__( force, conda_name, conda_version, - minimal, + empty_template, ) From c6cf507be77f5123a8582909b8e333177b96b0b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 3 Feb 2023 15:09:22 +0100 Subject: [PATCH 837/854] Update nf_core/__main__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 1372ff7fbc..841cb9f7e7 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -683,7 +683,7 @@ def remove(ctx, dir, tool): "--empty-template", is_flag=True, default=False, - help="Create a version of the template without TODOs or examples", + help="Create a module from the template without TODOs or examples", ) def create_module( ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template From 6ff70af488b642d43a27a3d89008fc1cac7ed0e4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 4 Feb 2023 19:33:35 +0100 Subject: [PATCH 838/854] Run new version of Black --- nf_core/bump_version.py | 2 -- nf_core/create.py | 1 - nf_core/download.py | 1 - nf_core/launch.py | 3 --- nf_core/lint/__init__.py | 1 - nf_core/lint/files_unchanged.py | 2 -- nf_core/lint/template_strings.py | 1 - nf_core/list.py | 1 - nf_core/modules/bump_versions.py | 1 - nf_core/utils.py | 2 -- tests/test_download.py | 1 - tests/test_test_utils.py | 1 - 12 files changed, 17 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 1f38775bb6..129016fa38 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -133,12 +133,10 @@ def update_file_version(filename, pipeline_obj, patterns): replacements = [] for pattern in patterns: - found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/create.py b/nf_core/create.py index 045c35d1b4..74c9df1b87 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -283,7 +283,6 @@ def render_template(self): # Set the paths to skip according to customization for template_fn_path_obj in template_files: - template_fn_path = str(template_fn_path_obj) # Skip files that are in the self.skip_paths list diff --git a/nf_core/download.py b/nf_core/download.py index 2f964b3afd..9d430e3352 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -506,7 +506,6 @@ def get_singularity_images(self): containers_download = [] containers_pull = [] for container in self.containers: - # Fetch the output and cached filenames for this container out_path, cache_path = self.singularity_image_filenames(container) diff --git a/nf_core/launch.py b/nf_core/launch.py index 87150172f7..648c8775f8 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -98,7 +98,6 @@ def __init__( self.cli_launch = True def launch_pipeline(self): - # Prompt for pipeline if not supplied and no web launch ID if self.pipeline is None and self.web_id is None: launch_type = questionary.select( @@ -461,7 +460,6 @@ def prompt_group(self, group_id, group_obj): answers = {} error_msgs = [] while not while_break: - if len(error_msgs) == 0: self.print_param_header(group_id, group_obj, True) @@ -698,7 +696,6 @@ def build_command(self): # Pipeline parameters if len(self.schema_obj.input_params) > 0: - # Write the user selection to a file and run nextflow with that if self.use_params_file: dump_json_with_prettier(self.params_out, self.schema_obj.input_params) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index b46399eb97..e014a933ea 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -434,7 +434,6 @@ def format_result(test_results): ) def _print_summary(self): - # Summary table summary_colour = "red" if len(self.failed) > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index cadced5483..c0be64d0d7 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -147,7 +147,6 @@ def _tf(file_path): # Files that must be completely unchanged from template for files in files_exact: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): @@ -177,7 +176,6 @@ def _tf(file_path): # Files that can be added to, but that must contain the template contents for files in files_partial: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index 436abe7b2b..fb1f0f32e5 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -24,7 +24,6 @@ def template_strings(self): # Loop through files, searching for string num_matches = 0 for fn in self.files: - # Skip binary files binary_ftypes = ["image", "application/java-archive"] (ftype, encoding) = mimetypes.guess_type(fn) diff --git a/nf_core/list.py b/nf_core/list.py index 53307ac9bd..77a9ac3919 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -328,7 +328,6 @@ def get_local_nf_workflow_details(self): """Get full details about a local cached workflow""" if self.local_path is None: - # Try to guess the local cache directory if len(os.environ.get("NXF_ASSETS", "")) > 0: nf_wfdir = os.path.join(os.environ.get("NXF_ASSETS"), self.full_name) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 7b9702622e..6d61d4c750 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -187,7 +187,6 @@ def bump_module_version(self, module): found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/utils.py b/nf_core/utils.py index b60f61fff0..55703ab450 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -872,7 +872,6 @@ def get_repo_releases_branches(pipeline, wfs): # Repo is a nf-core pipeline for wf in wfs.remote_workflows: if wf.full_name == pipeline or wf.name == pipeline: - # Set to full name just in case it didn't have the nf-core/ prefix pipeline = wf.full_name @@ -883,7 +882,6 @@ def get_repo_releases_branches(pipeline, wfs): # Arbitrary GitHub repo else: if pipeline.count("/") == 1: - # Looks like a GitHub address - try working with this repo log.debug( f"Pipeline '{pipeline}' not in nf-core, but looks like a GitHub address - fetching releases from API" diff --git a/tests/test_download.py b/tests/test_download.py index 4577a83992..e2ae882394 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -18,7 +18,6 @@ class DownloadTest(unittest.TestCase): - # # Tests for 'get_release_hash' # diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index ddf88ef74a..c4e3d49ae0 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -33,7 +33,6 @@ def test_tmp_folder_does_not_exist_after(): def test_set_wd(): - with tempfile.TemporaryDirectory() as tmpdirname: with set_wd(tmpdirname): context_wd = Path().resolve() From 2b75ef9c57fdc37b0d279583774f833b4df4075e Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 6 Feb 2023 14:30:56 +0100 Subject: [PATCH 839/854] add pre-commit conf to pipeline template, update pre-commit linter versions --- .pre-commit-config.yaml | 7 +++---- nf_core/.pre-commit-prettier-config.yaml | 2 +- nf_core/pipeline-template/.pre-commit-config.yaml | 5 +++++ 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 nf_core/pipeline-template/.pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eddb0f1048..b7aeeb5bc9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,13 @@ repos: - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.1.0 hooks: - id: black - language_version: python3.9 - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.6.2" + rev: "v2.7.1" hooks: - id: prettier diff --git a/nf_core/.pre-commit-prettier-config.yaml b/nf_core/.pre-commit-prettier-config.yaml index 80f75fb22c..0c31cdb99f 100644 --- a/nf_core/.pre-commit-prettier-config.yaml +++ b/nf_core/.pre-commit-prettier-config.yaml @@ -1,5 +1,5 @@ repos: - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.6.2" + rev: "v2.7.1" hooks: - id: prettier diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml new file mode 100644 index 0000000000..0c31cdb99f --- /dev/null +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.7.1" + hooks: + - id: prettier From 7c5d0966c955276efe213f38713da9c705f5c184 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 13:00:10 +0100 Subject: [PATCH 840/854] allowing new line before description field --- nf_core/module-template/modules/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 05ecb6f980..9b42bb3bdf 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,7 +1,7 @@ name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords -{%- endif -%} +{% endif -%} description: write your description here keywords: - sort From 18277d666c06a57baf3be9e8707c15afed9f4a80 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 14:55:26 +0100 Subject: [PATCH 841/854] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5dd860b3e..b5c5701120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ ### Modules -- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175)) +- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175) & [#2177](https://github.com/nf-core/tools/pull/2177)) ### Subworkflows From 53c874fc193ab73f7457d79f0239a53018393850 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 16:38:53 +0100 Subject: [PATCH 842/854] checkout to sha when getting list of available components --- nf_core/components/install.py | 6 +++--- nf_core/modules/modules_repo.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f9a16f73cf..cb6c59aec3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -169,19 +169,19 @@ def collect_and_verify_name(self, component, modules_repo): if component is None: component = questionary.autocomplete( f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:", - choices=sorted(modules_repo.get_avail_components(self.component_type)), + choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.sha)), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() # Check that the supplied name is an available module/subworkflow - if component and component not in modules_repo.get_avail_components(self.component_type): + if component and component not in modules_repo.get_avail_components(self.component_type, commit=self.sha): log.error( f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." ) log.info(f"Use the command 'nf-core {self.component_type} list' to view available software") return False - if not modules_repo.component_exists(component, self.component_type): + if not modules_repo.component_exists(component, self.component_type, commit=self.sha): warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" log.warning(warn_msg) return False diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 606514e55e..5f77148867 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -302,7 +302,7 @@ def checkout(self, commit): """ self.repo.git.checkout(commit) - def component_exists(self, component_name, component_type, checkout=True): + def component_exists(self, component_name, component_type, checkout=True, commit=None): """ Check if a module/subworkflow exists in the branch of the repo @@ -312,7 +312,7 @@ def component_exists(self, component_name, component_type, checkout=True): Returns: (bool): Whether the module/subworkflow exists in this branch of the repository """ - return component_name in self.get_avail_components(component_type, checkout=checkout) + return component_name in self.get_avail_components(component_type, checkout=checkout, commit=commit) def get_component_dir(self, component_name, component_type): """ @@ -449,7 +449,7 @@ def get_commit_info(self, sha): return message, date raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'") - def get_avail_components(self, component_type, checkout=True): + def get_avail_components(self, component_type, checkout=True, commit=None): """ Gets the names of the modules/subworkflows in the repository. They are detected by checking which directories have a 'main.nf' file @@ -459,6 +459,8 @@ def get_avail_components(self, component_type, checkout=True): """ if checkout: self.checkout_branch() + if commit is not None: + self.checkout(commit) # Get directory if component_type == "modules": directory = self.modules_dir From 7ea4b6e63481a08396212c0ba05468e8f5e32bf6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 Feb 2023 09:17:27 +0100 Subject: [PATCH 843/854] update gitpod dockerfile based on https://github.com/nextflow-io/training/blob/master/.github/gitpod.Dockerfile --- nf_core/gitpod/gitpod.Dockerfile | 38 ++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 07beed96ec..fa435c3612 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -2,6 +2,19 @@ FROM gitpod/workspace-base USER root +# Install util tools. +RUN apt-get update --quiet && \ + apt-get install --quiet --yes \ + apt-transport-https \ + apt-utils \ + sudo \ + git \ + less \ + wget \ + curl \ + tree \ + graphviz + # Install Conda RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ @@ -18,23 +31,20 @@ RUN chown -R gitpod:gitpod /opt/conda /usr/src/nf_core # Change user to gitpod USER gitpod - # Install nextflow, nf-core, Mamba, and pytest-workflow -RUN conda update -n base -c defaults conda && \ - conda config --add channels defaults && \ +RUN conda config --add channels defaults && \ conda config --add channels bioconda && \ conda config --add channels conda-forge && \ - conda install \ - openjdk=17.0.3 \ - nextflow=22.10.1 \ - nf-test=0.7.1-0 \ - pytest-workflow=1.6.0 \ - mamba=0.27.0 \ - pip=22.3 \ - black=22.10.0 \ - prettier=2.7.1 \ - -n base && \ - conda clean --all -f -y + conda config --set channel_priority strict && \ + conda install --quiet --yes --name base mamba && \ + mamba install --quiet --yes --name base \ + nextflow \ + nf-core \ + nf-test \ + black \ + prettier \ + pytest-workflow && \ + mamba clean --all -f -y # Install nf-core RUN python -m pip install . From b2f3698631721a09e83d4d315a5b29cec5b683c2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:35:25 +0100 Subject: [PATCH 844/854] skip modules whose name has changed --- nf_core/components/update.py | 15 ++++++++++++--- tests/subworkflows/update.py | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index ef645a5a1d..e640827594 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -386,7 +386,7 @@ def get_single_component_info(self, component): ) # Check that the supplied name is an available module/subworkflow - if component and component not in self.modules_repo.get_avail_components(self.component_type): + if component and component not in self.modules_repo.get_avail_components(self.component_type, commit=self.sha): raise LookupError( f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." f"Use the command 'nf-core {self.component_type} list remote' to view available software" @@ -879,8 +879,17 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up if m_update in updated: continue original_component_type, original_update_all = self._change_component_type("modules") - self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) - self._reset_component_type(original_component_type, original_update_all) + try: + self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) + except LookupError as e: + # If the module to be updated is not available, check if there has been a name change + if "not found in list of available" in str(e): + # Skip update, we check for name changes with manage_changes_in_linked_components in line #261 + pass + else: + raise + finally: + self._reset_component_type(original_component_type, original_update_all) def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update): """Check for linked components added or removed in the new subworkflow version""" diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 29e6cb1179..1d437f00a4 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -8,6 +8,7 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.modules.remove import ModuleRemove from nf_core.modules.update import ModuleUpdate from nf_core.subworkflows.update import SubworkflowUpdate From 39810e1d37149ead2905b7ee3b651fa9cb776266 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:47:12 +0100 Subject: [PATCH 845/854] change manage_changes order --- nf_core/components/update.py | 22 +++++++++++----------- tests/subworkflows/update.py | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index e640827594..ab4a6eedbb 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -896,17 +896,6 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw if self.component_type == "subworkflows": subworkflow_directory = Path(self.dir, self.component_type, self.modules_repo.repo_path, component) included_modules, included_subworkflows = get_components_to_install(subworkflow_directory) - # If a new module/subworkflow is included in the subworklfow and wasn't included before - for module in included_modules: - if module not in modules_to_update: - log.info(f"Installing newly included module '{module}' for '{component}'") - install_module_object = ComponentInstall(self.dir, "modules", installed_by=component) - install_module_object.install(module, silent=True) - for subworkflow in included_subworkflows: - if subworkflow not in subworkflows_to_update: - log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") - install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component) - install_subworkflow_object.install(subworkflow, silent=True) # If a module/subworkflow has been removed from the subworkflow for module in modules_to_update: if module not in included_modules: @@ -918,6 +907,17 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.") remove_subworkflow_object = ComponentRemove("subworkflows", self.dir) remove_subworkflow_object.remove(subworkflow, removed_by=component) + # If a new module/subworkflow is included in the subworklfow and wasn't included before + for module in included_modules: + if module not in modules_to_update: + log.info(f"Installing newly included module '{module}' for '{component}'") + install_module_object = ComponentInstall(self.dir, "modules", installed_by=component) + install_module_object.install(module, silent=True) + for subworkflow in included_subworkflows: + if subworkflow not in subworkflows_to_update: + log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") + install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component) + install_subworkflow_object.install(subworkflow, silent=True) def _change_component_type(self, new_component_type): original_component_type = self.component_type diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 1d437f00a4..698086e186 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -322,6 +322,10 @@ def test_update_change_of_included_modules(self): # Check that tabix/tabix is there assert "tabix/tabix" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/tabix").is_dir() + # Check that ensemblevep is not there but instead we have ensemblevep/vep (due to a file re-naming) + assert "ensemblvep" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert "ensemblvep/vep" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "ensemblvep/vep").is_dir() def cmp_component(dir1, dir2): From 297b2c8fd6398b0e1c16981616392308b2e4380a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:54:11 +0100 Subject: [PATCH 846/854] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c5701120..3b3d15f18a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] -- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) +- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) ### Linting @@ -16,6 +16,8 @@ ### Subworkflows +- Fixing problem when a module included in a subworkflow had a name change from TOOL to TOOL/SUBTOOL ([#2177](https://github.com/nf-core/tools/pull/2177)) + ### General ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From 53af36c2cccd3c87c9afc6de3af4a708890fa6d4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 9 Feb 2023 11:09:04 +0100 Subject: [PATCH 847/854] add 'subworkflows/' to pytest tag --- nf_core/components/components_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index eadb5ba0cb..c2b9abf569 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -180,7 +180,10 @@ def _run_pytests(self): console.rule(self.component_name, style="black") # Set pytest arguments - command_args = ["--tag", f"{self.component_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] + tag = self.component_name + if self.component_type == "subworkflows": + tag = "subworkflows/" + tag + command_args = ["--tag", f"{tag}", "--symlink", "--keep-workflow-wd", "--git-aware"] command_args += self.pytest_args # Run pytest From 2250919643065125f406c69ca28937df0b5b3c0d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 9 Feb 2023 11:11:18 +0100 Subject: [PATCH 848/854] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b3d15f18a..ac67ab7bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### Subworkflows - Fixing problem when a module included in a subworkflow had a name change from TOOL to TOOL/SUBTOOL ([#2177](https://github.com/nf-core/tools/pull/2177)) +- Fix `nf-core subworkflows test` not running subworkflow tests ([#2181](https://github.com/nf-core/tools/pull/2181)) ### General From cab29c86aa70a838c18f3f5a1f05b62cf7328b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 9 Feb 2023 11:40:26 +0100 Subject: [PATCH 849/854] Update nf_core/components/update.py --- nf_core/components/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index ab4a6eedbb..5f8a2129c5 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -884,7 +884,7 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up except LookupError as e: # If the module to be updated is not available, check if there has been a name change if "not found in list of available" in str(e): - # Skip update, we check for name changes with manage_changes_in_linked_components in line #261 + # Skip update, we check for name changes with manage_changes_in_linked_components pass else: raise From 4604e408a91bd3f37f2d3b480d0544e5ec9c18de Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 9 Feb 2023 11:57:11 +0100 Subject: [PATCH 850/854] add include statement to info command for installed components fix incorrect handling of locally installed modules --- nf_core/components/info.py | 40 ++++++++++++++++++++++++++++++++++- nf_core/components/install.py | 5 ++--- tests/modules/info.py | 2 +- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/nf_core/components/info.py b/nf_core/components/info.py index b89baaf99e..69b9e19b87 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -8,6 +8,7 @@ from rich.console import Group from rich.markdown import Markdown from rich.panel import Panel +from rich.syntax import Syntax from rich.table import Table from rich.text import Text @@ -128,6 +129,18 @@ def init_mod_name(self, component): choices=components, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() + else: + if self.repo_type == "pipeline": + # check if the module is locally installed + self.local_path = self.modules_json.get_all_components(self.component_type).get( + self.modules_repo.remote_url, {} + ) + for directory, comp in self.local_path: + if comp == component: + self.local_path = Path(self.component_type, directory, component) + break + if self.local_path: + self.local = True return component @@ -283,7 +296,7 @@ def generate_component_info_help(self): renderables.append(outputs_table) # Installation command - if self.remote_location: + if self.remote_location and not self.local: cmd_base = f"nf-core {self.component_type}" if self.remote_location != NF_CORE_MODULES_REMOTE: cmd_base = f"nf-core {self.component_type} --git-remote {self.remote_location}" @@ -291,4 +304,29 @@ def generate_component_info_help(self): Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.component}\n") ) + # Print include statement + if self.local_path: + install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path) + component_name = "_".join(self.component.upper().split("/")) + renderables.append( + Text.from_markup(f"\n [blue]Use the following statement to include this {self.component_type[:-1]}:") + ) + renderables.append( + Syntax( + f"include {{ {component_name} }} from '../{Path(install_folder, self.component).relative_to(self.dir)}/main'", + "groovy", + theme="ansi_dark", + padding=1, + ) + ) + if self.component_type == "subworkflows": + subworkflow_config = Path(install_folder, self.component, "nextflow.config").relative_to(self.dir) + if os.path.isfile(subworkflow_config): + renderables.append( + Text.from_markup("\n [blue]Add the following config statement to use this subworkflow:") + ) + renderables.append( + Syntax(f"includeConfig '{subworkflow_config}'", "groovy", theme="ansi_dark", padding=1) + ) + return Group(*renderables) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f9a16f73cf..90cdf7b6a9 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -1,6 +1,5 @@ import logging import os -import re from pathlib import Path import questionary @@ -74,10 +73,10 @@ def install(self, component, silent=False): ) # Set the install folder based on the repository name - install_folder = os.path.join(self.dir, self.component_type, self.modules_repo.repo_path) + install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path) # Compute the component directory - component_dir = os.path.join(install_folder, component) + component_dir = Path(install_folder, component) # Check that the component is not already installed component_not_installed = self.check_component_installed( diff --git a/tests/modules/info.py b/tests/modules/info.py index 6c5b1063f1..2dbd48b240 100644 --- a/tests/modules/info.py +++ b/tests/modules/info.py @@ -38,7 +38,6 @@ def test_modules_info_local(self): """Test getting info about a locally installed module""" self.mods_install.install("trimgalore") mods_info = nf_core.modules.ModuleInfo(self.pipeline_dir, "trimgalore") - mods_info.local = True mods_info_output = mods_info.get_component_info() console = Console(record=True) console.print(mods_info_output) @@ -47,6 +46,7 @@ def test_modules_info_local(self): assert "Module: trimgalore" in output assert "Inputs" in output assert "Outputs" in output + assert "Location" in output def test_modules_info_in_modules_repo(self): From 2d04d7294ad997a9787cd168be8850f7ea4ac4ea Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 9 Feb 2023 12:09:35 +0100 Subject: [PATCH 851/854] update changelog, use same way to generate self.local_path as in the rest of the code --- CHANGELOG.md | 2 ++ nf_core/components/info.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5dd860b3e..16900b7a59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ ### General +- `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). + ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 69b9e19b87..365e017ab1 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -132,12 +132,13 @@ def init_mod_name(self, component): else: if self.repo_type == "pipeline": # check if the module is locally installed - self.local_path = self.modules_json.get_all_components(self.component_type).get( + local_paths = self.modules_json.get_all_components(self.component_type).get( self.modules_repo.remote_url, {} ) - for directory, comp in self.local_path: + for directory, comp in local_paths: if comp == component: - self.local_path = Path(self.component_type, directory, component) + component_base_path = Path(self.dir, self.component_type) + self.local_path = Path(component_base_path, directory, self.component) break if self.local_path: self.local = True From 6ce1b3c72b146ebf296493facde2ac1cbf0df312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 9 Feb 2023 12:54:03 +0100 Subject: [PATCH 852/854] Update nf_core/components/info.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 365e017ab1..e4d8038b87 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -138,7 +138,7 @@ def init_mod_name(self, component): for directory, comp in local_paths: if comp == component: component_base_path = Path(self.dir, self.component_type) - self.local_path = Path(component_base_path, directory, self.component) + self.local_path = Path(component_base_path, directory, component) break if self.local_path: self.local = True From 7915b47b4672b8bb27d0bd996f7f2b5eb2b363a8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 22 Feb 2023 09:38:57 +0100 Subject: [PATCH 853/854] add repository_type to .nf-core.yml --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 60dc102996..5c47e815d9 100644 --- a/README.md +++ b/README.md @@ -1032,6 +1032,8 @@ To list subworkflows installed in a local pipeline directory you can use `nf-cor @@ -1056,6 +1058,8 @@ A subworkflow installed this way will be installed to the `./subworkflows/nf-cor ![`nf-core subworkflows install bam_rseqc`](docs/images/nf-core-subworkflows-install.svg) @@ -1075,6 +1079,8 @@ You can update subworkflows installed from a remote repository in your pipeline ![`nf-core subworkflows update --all --no-preview`](docs/images/nf-core-subworkflows-update.svg) @@ -1134,6 +1140,8 @@ To delete a subworkflow from your pipeline, run `nf-core subworkflows remove`. ![`nf-core subworkflows remove bam_rseqc`](docs/images/nf-core-subworkflows-remove.svg) @@ -1162,10 +1170,10 @@ The `nf-core subworkflows create` command will prompt you with the relevant ques -![`cd modules && nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --label process_low --meta --force`](docs/images/nf-core-subworkflows-create.svg) +![`cd modules && nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force`](docs/images/nf-core-subworkflows-create.svg) ### Create a subworkflow test config file @@ -1177,6 +1185,8 @@ After you have written a minimal Nextflow script to test your subworkflow in `/t working_dir: tmp/subworkflows extra_env: PROFILE: 'conda' +before_command: > + echo "repository_type: modules" >> .nf-core.yml --> ![`nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force`](docs/images/nf-core-subworkflows-create-test.svg) @@ -1192,6 +1202,8 @@ working_dir: tmp/subworkflows timeout: 30 extra_env: PROFILE: 'conda' +before_command: > + echo "repository_type: pipeline" >> .nf-core.yml --> ![`nf-core subworkflows test bam_rseqc --no-prompts`](docs/images/nf-core-subworkflows-test.svg) From c569daaed17d9a1e67a3887a92d45c8b4965904b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 22 Feb 2023 09:44:40 +0100 Subject: [PATCH 854/854] add repository_type to .nf-core.yml --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5c47e815d9..fa4fc96d0e 100644 --- a/README.md +++ b/README.md @@ -1047,6 +1047,8 @@ This shows documentation about the subworkflow on the command line, similar to w ![`nf-core subworkflows info bam_rseqc`](docs/images/nf-core-subworkflows-info.svg)