From 4368b2f73f88d2503175a9ff77e36017d0f2c95c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 22 Jul 2020 16:31:58 +0200 Subject: [PATCH 1/8] Start stripping out parameter docs from usage.md --- nf_core/launch.py | 1 + .../docs/usage.md | 98 ------------------- 2 files changed, 1 insertion(+), 98 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 4f224c4ad4..488d19c224 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -80,6 +80,7 @@ def __init__( "-name": { "type": "string", "description": "Unique name for this nextflow run", + "help_text": "If not specified, Nextflow will automatically generate a random mnemonic.", "pattern": "^[a-zA-Z0-9-_]+$", }, "-profile": {"type": "string", "description": "Configuration profile"}, diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index c13c735b7b..8fa0b6a7dd 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -118,86 +118,6 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof * A profile with a complete configuration for automated testing * Includes links to test data so needs no other parameters - - -### `--input` - -Use this to specify the location of your input FastQ files. For example: - -```bash ---input 'path/to/data/sample_*_{1,2}.fastq' -``` - -Please note the following requirements: - -1. The path must be enclosed in quotes -2. The path must have at least one `*` wildcard character -3. When using the pipeline with paired end data, the path must use `{1,2}` notation to specify read pairs. - -If left unspecified, a default pattern is used: `data/*{1,2}.fastq.gz` - -### `--single_end` - -By default, the pipeline expects paired-end data. If you have single-end data, you need to specify `--single_end` on the command line when you launch the pipeline. A normal glob pattern, enclosed in quotation marks, can then be used for `--input`. For example: - -```bash ---single_end --input '*.fastq' -``` - -It is not possible to run a mixture of single-end and paired-end files in one run. - -## Reference genomes - -The pipeline config files come bundled with paths to the illumina iGenomes reference index files. If running with docker or AWS, the configuration is set up to use the [AWS-iGenomes](https://ewels.github.io/AWS-iGenomes/) resource. - -### `--genome` (using iGenomes) - -There are 31 different species supported in the iGenomes references. To run the pipeline, you must specify which to use with the `--genome` flag. - -You can find the keys to specify the genomes in the [iGenomes config file](https://github.com/{{ cookiecutter.name }}/blob/master/conf/igenomes.config). Common genomes that are supported are: - -* Human - * `--genome GRCh37` -* Mouse - * `--genome GRCm38` -* _Drosophila_ - * `--genome BDGP6` -* _S. cerevisiae_ - * `--genome 'R64-1-1'` - -> There are numerous others - check the config file for more. - -Note that you can use the same configuration setup to save sets of reference files for your own use, even if they are not part of the iGenomes resource. See the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for instructions on where to save such a file. - -The syntax for this reference configuration is as follows: - - - -```nextflow -params { - genomes { - 'GRCh37' { - fasta = '' // Used if no star index given - } - // Any number of additional genomes, key is used with --genome - } -} -``` - - - -### `--fasta` - -If you prefer, you can specify the full path to your reference genome when you run the pipeline: - -```bash ---fasta '[path to Fasta reference]' -``` - -### `--igenomes_ignore` - -Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`. - ## Job resources ### Automatic resubmission @@ -232,16 +152,6 @@ Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a ## Other command line parameters - - -### `--outdir` - -The output directory where the results will be saved. - -### `--publish_dir_mode` - -Value passed to Nextflow [`publishDir`](https://www.nextflow.io/docs/latest/process.html#publishdir) directive for publishing results in the output directory. Available: 'symlink', 'rellink', 'link', 'copy', 'copyNoFollow' and 'move' (Default: 'copy'). - ### `--email` Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run. @@ -254,14 +164,6 @@ This works exactly as with `--email`, except emails are only sent if the workflo Threshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB). -### `-name` - -Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic. - -This is used in the MultiQC report (if not default) and in the summary HTML / e-mail (always). - -**NB:** Single hyphen (core Nextflow option) - ### `-resume` Specify this when restarting a pipeline. Nextflow will used cached results from any pipeline steps where the inputs are the same, continuing from where it got to previously. From e0c284a3b2efacf2229d3e043f83adfb5d39b67d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 12:55:38 +0200 Subject: [PATCH 2/8] Strip more usage docs and update schema See nf-core/tools#647 --- .../docs/usage.md | 119 +++--------------- .../nextflow_schema.json | 80 +++++++----- 2 files changed, 67 insertions(+), 132 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 8fa0b6a7dd..845ef29705 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -87,7 +87,9 @@ First, go to the [{{ cookiecutter.name }} releases page](https://github.com/{{ c This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. -## Main arguments +## Core Nextflow arguments + +> **NB:** These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). ### `-profile` @@ -118,121 +120,32 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof * A profile with a complete configuration for automated testing * Includes links to test data so needs no other parameters -## Job resources - -### Automatic resubmission - -Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with an error code of `143` (exceeded requested resources) it will automatically resubmit with higher requests (2 x original, then 3 x original). If it still fails after three times then the pipeline is stopped. - -### Custom resource requests - -Wherever process-specific requirements are set in the pipeline, the default value can be changed by creating a custom config file. See the files hosted at [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf) for examples. - -If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. - -If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack). - -## AWS Batch specific parameters - -Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use [`-profile awsbatch`](https://github.com/nf-core/configs/blob/master/conf/awsbatch.config) and then specify all of the following parameters. - -### `--awsqueue` - -The JobQueue that you intend to use on AWS Batch. - -### `--awsregion` - -The AWS region in which to run your job. Default is set to `eu-west-1` but can be adjusted to your needs. - -### `--awscli` - -The [AWS CLI](https://www.nextflow.io/docs/latest/awscloud.html#aws-cli-installation) path in your custom AMI. Default: `/home/ec2-user/miniconda/bin/aws`. - -Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a S3 storage bucket of your choice - you'll get an error message notifying you if you didn't. - -## Other command line parameters - -### `--email` - -Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run. - -### `--email_on_fail` - -This works exactly as with `--email`, except emails are only sent if the workflow is not successful. - -### `--max_multiqc_email_size` - -Threshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB). - ### `-resume` Specify this when restarting a pipeline. Nextflow will used cached results from any pipeline steps where the inputs are the same, continuing from where it got to previously. You can also supply a run name to resume a specific run: `-resume [run-name]`. Use the `nextflow log` command to show previous run names. -**NB:** Single hyphen (core Nextflow option) - ### `-c` -Specify the path to a specific config file (this is a core NextFlow command). - -**NB:** Single hyphen (core Nextflow option) - -Note - you can use this to override pipeline defaults. +Specify the path to a specific config file (this is a core NextFlow command). See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information. -### `--custom_config_version` - -Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default: `master`. - -```bash -## Download and use config file with following git commid id ---custom_config_version d52db660777c4bf36546ddb188ec530c3ada1b96 -``` +### Custom resource requests -### `--custom_config_base` +Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with an error code of `143` (exceeded requested resources) it will automatically resubmit with higher requests (2 x original, then 3 x original). If it still fails after three times then the pipeline is stopped. -If you're running offline, nextflow will not be able to fetch the institutional config files -from the internet. If you don't need them, then this is not a problem. If you do need them, -you should download the files from the repo and tell nextflow where to find them with the -`custom_config_base` option. For example: +Whilst these default requirements will hopefully work for most people with most data, you may find that you want to customise the compute resources that the pipeline requests. You can do this by creating a custom config file. For example, to give the workflow process `star` 32GB of memory, you could use the following config: -```bash -## Download and unzip the config files -cd /path/to/my/configs -wget https://github.com/nf-core/configs/archive/master.zip -unzip master.zip - -## Run the pipeline -cd /path/to/my/data -nextflow run /path/to/pipeline/ --custom_config_base /path/to/my/configs/configs-master/ +```nextflow +process { + withName: star { + memory = 32.GB + } +} ``` -> Note that the nf-core/tools helper package has a `download` command to download all required pipeline -> files + singularity containers + institutional configs in one go for you, to make this process easier. - -### `--max_memory` - -Use to set a top-limit for the default memory requirement for each process. -Should be a string in the format integer-unit. eg. `--max_memory '8.GB'` - -### `--max_time` - -Use to set a top-limit for the default time requirement for each process. -Should be a string in the format integer-unit. eg. `--max_time '2.h'` - -### `--max_cpus` +See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more information. -Use to set a top-limit for the default CPU requirement for each process. -Should be a string in the format integer-unit. eg. `--max_cpus 1` - -### `--plaintext_email` - -Set to receive plain-text e-mails instead of HTML formatted. - -### `--monochrome_logs` - -Set to disable colourful command line output and live life in monochrome. - -### `--multiqc_config` +If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. -Specify a path to a custom MultiQC configuration file. +If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json index d5c93776db..646799a6ab 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json @@ -18,14 +18,13 @@ "type": "string", "fa_icon": "fas fa-dna", "description": "Input FastQ files.", - "help_text": "A glob pattern for input FastQ files. Should include at least one asterisk (*). For paired-end data, should contain curly brackets with two patterns differentiating the paired reads e.g. `*_R{1,2}.fastq.gz`" + "help_text": "Use this to specify the location of your input FastQ files. For example:\n\n```bash\n--input 'path/to/data/sample_*_{1,2}.fastq'\n```\n\nPlease note the following requirements:\n\n1. The path must be enclosed in quotes\n2. The path must have at least one `*` wildcard character\n3. When using the pipeline with paired end data, the path must use `{1,2}` notation to specify read pairs.\n\nIf left unspecified, a default pattern is used: `data/*{1,2}.fastq.gz`" }, "single_end": { "type": "boolean", "description": "Specifies that the input is single-end reads.", "fa_icon": "fas fa-align-center", - "default": false, - "help_text": "By default, the pipeline expects paired-end data. If you have single-end data, specify this parameter on the command line when you launch the pipeline. It is not possible to run a mixture of single-end and paired-end files in one run." + "help_text": "By default, the pipeline expects paired-end data. If you have single-end data, you need to specify `--single_end` on the command line when you launch the pipeline. A normal glob pattern, enclosed in quotation marks, can then be used for `--input`. For example:\n\n```bash\n--single_end --input '*.fastq'\n```\n\nIt is not possible to run a mixture of single-end and paired-end files in one run." }, "outdir": { "type": "string", @@ -37,7 +36,7 @@ "type": "string", "description": "Email address for completion summary.", "fa_icon": "fas fa-envelope", - "help_text": "An email address to send a summary email to when the pipeline is completed.", + "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.", "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$" } } @@ -65,15 +64,13 @@ "description": "Directory / URL base for iGenomes references.", "default": "s3://ngi-igenomes/igenomes/", "fa_icon": "fas fa-cloud-download-alt", - "hidden": true, - "help_text": "" + "hidden": true }, "igenomes_ignore": { "type": "boolean", "description": "Do not load the iGenomes reference config.", "fa_icon": "fas fa-ban", "hidden": true, - "default": false, "help_text": "Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`." } } @@ -89,8 +86,7 @@ "type": "boolean", "description": "Display help text.", "hidden": true, - "fa_icon": "fas fa-question-circle", - "default": false + "fa_icon": "fas fa-question-circle" }, "publish_dir_mode": { "type": "string", @@ -121,15 +117,14 @@ "fa_icon": "fas fa-exclamation-triangle", "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$", "hidden": true, - "help_text": "An email address to send a summary email to when the pipeline is completed - ONLY sent if the pipeline does not exit successfully." + "help_text": "This works exactly as with `--email`, except emails are only sent if the workflow is not successful." }, "plaintext_email": { "type": "boolean", "description": "Send plain-text email instead of HTML.", "fa_icon": "fas fa-remove-format", "hidden": true, - "default": false, - "help_text": "" + "help_text": "Set to receive plain-text e-mails instead of HTML formatted." }, "max_multiqc_email_size": { "type": "string", @@ -137,30 +132,27 @@ "default": "25.MB", "fa_icon": "fas fa-file-upload", "hidden": true, - "help_text": "" + "help_text": "If file generated by pipeline exceeds the threshold, it will not be attached." }, "monochrome_logs": { "type": "boolean", "description": "Do not use coloured log outputs.", "fa_icon": "fas fa-palette", "hidden": true, - "default": false, - "help_text": "" + "help_text": "Set to disable colourful command line output and live life in monochrome." }, "multiqc_config": { "type": "string", "description": "Custom config file to supply to MultiQC.", "fa_icon": "fas fa-cog", - "hidden": true, - "help_text": "" + "hidden": true }, "tracedir": { "type": "string", "description": "Directory to keep pipeline Nextflow logs and reports.", "default": "${params.outdir}/pipeline_info", "fa_icon": "fas fa-cogs", - "hidden": true, - "help_text": "" + "hidden": true } } }, @@ -210,45 +202,72 @@ "default": "master", "hidden": true, "fa_icon": "fas fa-users-cog", - "help_text": "" + "help_text": "Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default: `master`.\n\n```bash\n## Download and use config file with following git commid id\n--custom_config_version d52db660777c4bf36546ddb188ec530c3ada1b96\n```" }, "custom_config_base": { "type": "string", "description": "Base directory for Institutional configs.", "default": "https://raw.githubusercontent.com/nf-core/configs/master", "hidden": true, - "help_text": "If you're running offline, Nextflow will not be able to fetch the institutional config files from the internet. If you don't need them, then this is not a problem. If you do need them, you should download the files from the repo and tell Nextflow where to find them with this parameter.", + "help_text": "If you're running offline, nextflow will not be able to fetch the institutional config files from the internet. If you don't need them, then this is not a problem. If you do need them, you should download the files from the repo and tell nextflow where to find them with the `custom_config_base` option. For example:\n\n```bash\n## Download and unzip the config files\ncd /path/to/my/configs\nwget https://github.com/nf-core/configs/archive/master.zip\nunzip master.zip\n\n## Run the pipeline\ncd /path/to/my/data\nnextflow run /path/to/pipeline/ --custom_config_base /path/to/my/configs/configs-master/\n```\n\n> Note that the nf-core/tools helper package has a `download` command to download all required pipeline files + singularity containers + institutional configs in one go for you, to make this process easier.", "fa_icon": "fas fa-users-cog" }, "hostnames": { "type": "string", "description": "Institutional configs hostname.", "hidden": true, - "fa_icon": "fas fa-users-cog", - "help_text": "" + "fa_icon": "fas fa-users-cog" }, "config_profile_description": { "type": "string", "description": "Institutional config description.", "hidden": true, - "fa_icon": "fas fa-users-cog", - "help_text": "" + "fa_icon": "fas fa-users-cog" }, "config_profile_contact": { "type": "string", "description": "Institutional config contact information.", "hidden": true, - "fa_icon": "fas fa-users-cog", - "help_text": "" + "fa_icon": "fas fa-users-cog" }, "config_profile_url": { "type": "string", "description": "Institutional config URL link.", "hidden": true, - "fa_icon": "fas fa-users-cog", - "help_text": "" + "fa_icon": "fas fa-users-cog" } } + }, + "aws_batch_specific_parameters": { + "title": "AWS Batch specific parameters", + "type": "object", + "description": "", + "default": "", + "properties": { + "awsqueue": { + "type": "string", + "description": "The JobQueue that you intend to use on AWS Batch.", + "fa_icon": "fab fa-aws", + "hidden": true + }, + "awsregion": { + "type": "string", + "description": "The AWS region in which to run your job.", + "fa_icon": "fab fa-aws", + "hidden": true, + "default": "eu-west-1" + }, + "awscli": { + "type": "string", + "help_text": "Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a S3 storage bucket of your choice - you'll get an error message notifying you if you didn't.\n\nRead more: [AWS CLI docs](https://www.nextflow.io/docs/latest/awscloud.html#aws-cli-installation).", + "description": "The AWS CLI path in your custom AMI.", + "fa_icon": "fab fa-aws", + "hidden": true, + "default": "/home/ec2-user/miniconda/bin/aws" + } + }, + "fa_icon": "fab fa-aws", + "help_text": "Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use [`-profile awsbatch`](https://github.com/nf-core/configs/blob/master/conf/awsbatch.config) and then specify all of the following parameters." } }, "allOf": [ @@ -266,6 +285,9 @@ }, { "$ref": "#/definitions/institutional_config_options" + }, + { + "$ref": "#/definitions/aws_batch_specific_parameters" } ] } From 37d1d6f2b5f303c15c3561e39e7c405cc9831f3d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 12:57:58 +0200 Subject: [PATCH 3/8] Strip aws_batch params from schema People can read about these on nf-core/configs I think, as that's where the config profile is kept. --- .../nextflow_schema.json | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json index 646799a6ab..671e0301b9 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json @@ -237,37 +237,6 @@ "fa_icon": "fas fa-users-cog" } } - }, - "aws_batch_specific_parameters": { - "title": "AWS Batch specific parameters", - "type": "object", - "description": "", - "default": "", - "properties": { - "awsqueue": { - "type": "string", - "description": "The JobQueue that you intend to use on AWS Batch.", - "fa_icon": "fab fa-aws", - "hidden": true - }, - "awsregion": { - "type": "string", - "description": "The AWS region in which to run your job.", - "fa_icon": "fab fa-aws", - "hidden": true, - "default": "eu-west-1" - }, - "awscli": { - "type": "string", - "help_text": "Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a S3 storage bucket of your choice - you'll get an error message notifying you if you didn't.\n\nRead more: [AWS CLI docs](https://www.nextflow.io/docs/latest/awscloud.html#aws-cli-installation).", - "description": "The AWS CLI path in your custom AMI.", - "fa_icon": "fab fa-aws", - "hidden": true, - "default": "/home/ec2-user/miniconda/bin/aws" - } - }, - "fa_icon": "fab fa-aws", - "help_text": "Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use [`-profile awsbatch`](https://github.com/nf-core/configs/blob/master/conf/awsbatch.config) and then specify all of the following parameters." } }, "allOf": [ @@ -285,9 +254,6 @@ }, { "$ref": "#/definitions/institutional_config_options" - }, - { - "$ref": "#/definitions/aws_batch_specific_parameters" } ] } From f3aea54f1b0cb4fcb67b3f2da1d15162971852c3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 13:04:05 +0200 Subject: [PATCH 4/8] Strip usage ToC and rewrite docs about detaching nextflow from the terminal --- .../docs/usage.md | 70 +++++-------------- 1 file changed, 19 insertions(+), 51 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 845ef29705..414b50d121 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -1,55 +1,5 @@ -# {{ cookiecutter.name }}: Usage - -## Table of contents - -* [Table of contents](#table-of-contents) -* [Introduction](#introduction) -* [Running the pipeline](#running-the-pipeline) - * [Updating the pipeline](#updating-the-pipeline) - * [Reproducibility](#reproducibility) -* [Main arguments](#main-arguments) - * [`-profile`](#-profile) - * [`--input`](#--input) - * [`--single_end`](#--single_end) -* [Reference genomes](#reference-genomes) - * [`--genome` (using iGenomes)](#--genome-using-igenomes) - * [`--fasta`](#--fasta) - * [`--igenomes_ignore`](#--igenomes_ignore) -* [Job resources](#job-resources) - * [Automatic resubmission](#automatic-resubmission) - * [Custom resource requests](#custom-resource-requests) -* [AWS Batch specific parameters](#aws-batch-specific-parameters) - * [`--awsqueue`](#--awsqueue) - * [`--awsregion`](#--awsregion) - * [`--awscli`](#--awscli) -* [Other command line parameters](#other-command-line-parameters) - * [`--outdir`](#--outdir) - * [`--publish_dir_mode`](#--publish_dir_mode) - * [`--email`](#--email) - * [`--email_on_fail`](#--email_on_fail) - * [`--max_multiqc_email_size`](#--max_multiqc_email_size) - * [`-name`](#-name) - * [`-resume`](#-resume) - * [`-c`](#-c) - * [`--custom_config_version`](#--custom_config_version) - * [`--custom_config_base`](#--custom_config_base) - * [`--max_memory`](#--max_memory) - * [`--max_time`](#--max_time) - * [`--max_cpus`](#--max_cpus) - * [`--plaintext_email`](#--plaintext_email) - * [`--monochrome_logs`](#--monochrome_logs) - * [`--multiqc_config`](#--multiqc_config) - ## Introduction -Nextflow handles job submissions on SLURM or other environments, and supervises running the jobs. Thus the Nextflow process must run until the pipeline is finished. We recommend that you put the process running in the background through `screen` / `tmux` or similar tool. Alternatively you can run nextflow within a cluster job submitted your job scheduler. - -It is recommended to limit the Nextflow Java virtual machines memory. We recommend adding the following line to your environment (typically in `~/.bashrc` or `~./bash_profile`): - -```bash -NXF_OPTS='-Xms1g -Xmx4g' -``` - ## Running the pipeline @@ -130,7 +80,7 @@ You can also supply a run name to resume a specific run: `-resume [run-name]`. U Specify the path to a specific config file (this is a core NextFlow command). See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information. -### Custom resource requests +#### Custom resource requests Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with an error code of `143` (exceeded requested resources) it will automatically resubmit with higher requests (2 x original, then 3 x original). If it still fails after three times then the pipeline is stopped. @@ -149,3 +99,21 @@ See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). + +### `-bg` + +Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. + +The Nextflow `-bg` flag launches Nextflow in the background, detached from your terminal so that the workflow does not stop if you log out of your session. The logs are saved to a file. + +Alternatively, you can use `screen` / `tmux` or similar tool to create a detached session which you can log back into at a later time. +Some HPC setups also allow you to run nextflow within a cluster job submitted your job scheduler (from where it submits more jobs). + +#### Nextflow memory requirements + +In some cases, the Nextflow Java virtual machines can start to request a large amount of memory. +We recommend adding the following line to your environment to limit this (typically in `~/.bashrc` or `~./bash_profile`): + +```bash +NXF_OPTS='-Xms1g -Xmx4g' +``` From 4e28e9727367e1020b92412a312ff0e5704823c8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 13:07:11 +0200 Subject: [PATCH 5/8] Add back the h1 to usage.md --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 414b50d121..90a7c48fe0 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -1,3 +1,5 @@ +# {{ cookiecutter.name }}: Usage + ## Introduction @@ -100,7 +102,7 @@ If you are likely to be running `nf-core` pipelines regularly it may be a good i If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). -### `-bg` +### Running in the background Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. From f3183de0ad4a761149c51d91cd77ad8487497026 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 13:08:40 +0200 Subject: [PATCH 6/8] Reword the usage.md TODO --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 90a7c48fe0..3517a308c9 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -2,7 +2,7 @@ ## Introduction - + ## Running the pipeline From 15b8294ae635be01999f975269eef0bf4085a2b8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 14:06:30 +0200 Subject: [PATCH 7/8] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json Co-authored-by: Gisela Gabernet --- .../{{cookiecutter.name_noslash}}/nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json index 671e0301b9..9b83fd4847 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json @@ -202,7 +202,7 @@ "default": "master", "hidden": true, "fa_icon": "fas fa-users-cog", - "help_text": "Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default: `master`.\n\n```bash\n## Download and use config file with following git commid id\n--custom_config_version d52db660777c4bf36546ddb188ec530c3ada1b96\n```" + "help_text": "Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default: `master`.\n\n```bash\n## Download and use config file with following git commit id\n--custom_config_version d52db660777c4bf36546ddb188ec530c3ada1b96\n```" }, "custom_config_base": { "type": "string", From cb72dda611e2074f36266cfa7a2c9648da0d9ab3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 30 Jul 2020 14:08:34 +0200 Subject: [PATCH 8/8] Schema - iGenomes - add link to website docs --- .../{{cookiecutter.name_noslash}}/nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json index 9b83fd4847..b12212e80b 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json @@ -51,7 +51,7 @@ "type": "string", "description": "Name of iGenomes reference.", "fa_icon": "fas fa-book", - "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`." + "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`.\n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details." }, "fasta": { "type": "string",