diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 3432a8f9f..68691263b 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -1,11 +1,12 @@ name: Create a pipeline and lint it on: [push, pull_request] +# Uncomment if we need an edge release of Nextflow again +# env: NXF_EDGE: 1 + jobs: MakeTestWorkflow: runs-on: ubuntu-latest - env: - NXF_VER: 21.03.0-edge steps: - uses: actions/checkout@v2 name: Check out source-code repository diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b9c638142..263a4f565 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -9,11 +9,12 @@ on: paths: - "**.py" +# Uncomment if we need an edge release of Nextflow again +# env: NXF_EDGE: 1 + jobs: pytest: runs-on: ubuntu-latest - env: - NXF_VER: 21.03.0-edge strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index d958fbee0..a0beb2363 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -4,8 +4,8 @@ on: types: [published] workflow_dispatch: -env: - NXF_VER: 21.03.0-edge +# Uncomment if we need an edge release of Nextflow again +# env: NXF_EDGE: 1 jobs: get-pipelines: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8104d5e1d..a250b2d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ * Merge markers lint test - ignore binary files, allow config to ignore specific files [[#1040](https://github.com/nf-core/tools/pull/1040)] * New lint test to check if params in `nextflow config` are mentioned in `main.nf` [[#1038](https://github.com/nf-core/tools/issues/1038)] * New modules lint test comparing the `functions.nf` file to the template version +* Added lint checks for missing parameter description and parameters outside of groups [[#1017](https://github.com/nf-core/tools/issues/1017)] +* Added fix to remove warnings about params that get converted from camelCase to camel-case [[#1035](https://github.com/nf-core/tools/issues/1035)] +* Use latest stable Nextflow version `21.04.0` for CI tests instead of the `-edge` release +* Fix bug in `nf-core download` where image names were getting a hyphen in `nf-core` which was breaking things. ### Template diff --git a/docs/api/_src/pipeline_lint_tests/schema_description.rst b/docs/api/_src/pipeline_lint_tests/schema_description.rst new file mode 100644 index 000000000..8733e203e --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/schema_description.rst @@ -0,0 +1,4 @@ +schema_description +=========== + +.. automethod:: nf_core.lint.PipelineLint.schema_description diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 7788c9b55..045682df3 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -144,8 +144,8 @@ def bump_nextflow_version(pipeline_obj, new_version): pipeline_obj, [ ( - r"nxf_ver: \[[\'\"]?{}[\'\"]?, '21.03.0-edge'\]".format(current_version.replace(".", r"\.")), - "nxf_ver: ['{}', '21.03.0-edge']".format(new_version), + r"nxf_ver: \[[\'\"]?{}[\'\"]?, ''\]".format(current_version.replace(".", r"\.")), + "nxf_ver: ['{}', '']".format(new_version), ) ], ) diff --git a/nf_core/download.py b/nf_core/download.py index 78c666610..0ce7891ca 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -605,8 +605,6 @@ def singularity_image_filenames(self, container): out_name = out_name[:-4] # Strip : and / characters out_name = out_name.replace("/", "-").replace(":", "-") - # Stupid Docker Hub not allowing hyphens - out_name = out_name.replace("nfcore", "nf-core") # Add file extension out_name = out_name + extension diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index a256fab6b..9edd9278b 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -113,6 +113,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .readme import readme from .schema_lint import schema_lint from .schema_params import schema_params + from .schema_description import schema_description from .template_strings import template_strings from .version_consistency import version_consistency @@ -147,6 +148,7 @@ def __init__(self, wf_path, release_mode=False, fix=(), key=(), fail_ignored=Fal "template_strings", "schema_lint", "schema_params", + "schema_description", "actions_schema_validation", "merge_markers", ] diff --git a/nf_core/lint/schema_description.py b/nf_core/lint/schema_description.py new file mode 100644 index 000000000..f1377053e --- /dev/null +++ b/nf_core/lint/schema_description.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +from logging import warn +import nf_core.schema + + +def schema_description(self): + """Check that every parameter in the schema has a description + + The ``nextflow_schema.json`` pipeline schema should describe every flat parameter + Furthermore warns about parameters outside of groups + + * Warning: Parameters in ``nextflow_schema.json`` without a description + * Warning: Parameters in ``nextflow_schema.json`` that are defined outside of a group + """ + passed = [] + warned = [] + ignored = [] + + # First, get the top-level config options for the pipeline + # Schema object already created in the `schema_lint` test + self.schema_obj = nf_core.schema.PipelineSchema() + self.schema_obj.get_schema_path(self.wf_path) + self.schema_obj.get_wf_params() + self.schema_obj.no_prompts = True + self.schema_obj.load_lint_schema() + + # Get parameters that should be ignored according to the linting config + ignore_params = self.lint_config.get("schema_description", []) + + # Get ungrouped params + if "properties" in self.schema_obj.schema.keys(): + ungrouped_params = self.schema_obj.schema["properties"].keys() + for up in ungrouped_params: + if up in ignore_params: + ignored.append(f"Ignored ungrouped param in schema: `{up}`") + else: + warned.append(f"Ungrouped param in schema: `{up}`") + + # Iterate over groups and add warning for parameters without a description + for group_key in self.schema_obj.schema["definitions"].keys(): + group = self.schema_obj.schema["definitions"][group_key] + for param_key, param in group["properties"].items(): + if param_key in ignore_params: + ignored.append(f"Ignoring description check for param in schema: `{param_key}`") + continue + if "description" not in param.keys(): + warned.append(f"No description provided in schema for parameter: `{param_key}`") + + for ip in ignore_params: + ignored.append(f"Parameter is ignored: `{ip}`") + + return {"passed": passed, "warned": warned, "ignored": ignored} diff --git a/nf_core/lint/schema_params.py b/nf_core/lint/schema_params.py index 580e9129d..20c962c22 100644 --- a/nf_core/lint/schema_params.py +++ b/nf_core/lint/schema_params.py @@ -17,10 +17,11 @@ def schema_params(self): failed = [] # First, get the top-level config options for the pipeline - # Schema object already created in the `schema_lint` test + self.schema_obj = nf_core.schema.PipelineSchema() self.schema_obj.get_schema_path(self.wf_path) self.schema_obj.get_wf_params() self.schema_obj.no_prompts = True + self.schema_obj.load_lint_schema() # Remove any schema params not found in the config removed_params = self.schema_obj.remove_schema_notfound_configs() diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 2f387022e..2c9c12aa5 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -8,6 +8,9 @@ on: release: types: [published] +# Uncomment if we need an edge release of Nextflow again +# env: NXF_EDGE: 1 + jobs: test: name: Run workflow tests @@ -20,7 +23,7 @@ jobs: strategy: matrix: # Nextflow versions: check pipeline minimum and current latest - nxf_ver: ['20.04.0', '21.03.0-edge'] + nxf_ver: ['20.04.0', ''] steps: - name: Check out pipeline code uses: actions/checkout@v2 diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index d591b434c..52ee73043 100644 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -112,8 +112,14 @@ class NfcoreSchema { } // unexpected params def params_ignore = params.schema_ignore_params.split(',') + 'schema_ignore_params' - if (!expectedParams.contains(specifiedParam) && !params_ignore.contains(specifiedParam)) { - unexpectedParams.push(specifiedParam) + def expectedParamsLowerCase = expectedParams.collect{ it.replace("-", "").toLowerCase() } + def specifiedParamLowerCase = specifiedParam.replace("-", "").toLowerCase() + if (!expectedParams.contains(specifiedParam) && !params_ignore.contains(specifiedParam) && !expectedParamsLowerCase.contains(specifiedParamLowerCase)) { + // Temporarily remove camelCase/camel-case params #1035 + def unexpectedParamsLowerCase = unexpectedParams.collect{ it.replace("-", "").toLowerCase()} + if (!unexpectedParamsLowerCase.contains(specifiedParamLowerCase)){ + unexpectedParams.push(specifiedParam) + } } }