Merge branch 'master' into prompts

.github/workflows/create-lint-wf.yml

@@ -1,11 +1,12 @@
 name: Create a pipeline and lint it
 on: [push, pull_request]
 
+# Uncomment if we need an edge release of Nextflow again
+# env: NXF_EDGE: 1
+
 jobs:
   MakeTestWorkflow:
     runs-on: ubuntu-latest
-    env:
-      NXF_VER: 21.03.0-edge
     steps:
       - uses: actions/checkout@v2
         name: Check out source-code repository

.github/workflows/pytest.yml

@@ -9,11 +9,12 @@ on:
     paths:
       - "**.py"
 
+# Uncomment if we need an edge release of Nextflow again
+# env: NXF_EDGE: 1
+
 jobs:
   pytest:
     runs-on: ubuntu-latest
-    env:
-      NXF_VER: 21.03.0-edge
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]

.github/workflows/sync.yml

@@ -4,8 +4,8 @@ on:
     types: [published]
   workflow_dispatch:
 
-env:
-  NXF_VER: 21.03.0-edge
+# Uncomment if we need an edge release of Nextflow again
+# env: NXF_EDGE: 1
 
 jobs:
   get-pipelines:

CHANGELOG.md

@@ -17,6 +17,10 @@
 * Merge markers lint test - ignore binary files, allow config to ignore specific files [[#1040](https://github.com/nf-core/tools/pull/1040)]
 * New lint test to check if params in `nextflow config` are mentioned in `main.nf` [[#1038](https://github.com/nf-core/tools/issues/1038)]
 * New modules lint test comparing the `functions.nf` file to the template version
+* Added lint checks for missing parameter description and parameters outside of groups [[#1017](https://github.com/nf-core/tools/issues/1017)]
+* Added fix to remove warnings about params that get converted from camelCase to camel-case [[#1035](https://github.com/nf-core/tools/issues/1035)]
+* Use latest stable Nextflow version `21.04.0` for CI tests instead of the `-edge` release
+* Fix bug in `nf-core download` where image names were getting a hyphen in `nf-core` which was breaking things.
 
 ### Template
 

docs/api/_src/pipeline_lint_tests/schema_description.rst

@@ -0,0 +1,4 @@
+schema_description
+===========
+
+.. automethod:: nf_core.lint.PipelineLint.schema_description

nf_core/bump_version.py

@@ -144,8 +144,8 @@ def bump_nextflow_version(pipeline_obj, new_version):
         pipeline_obj,
         [
             (
-                r"nxf_ver: \[[\'\"]?{}[\'\"]?, '21.03.0-edge'\]".format(current_version.replace(".", r"\.")),
-                "nxf_ver: ['{}', '21.03.0-edge']".format(new_version),
+                r"nxf_ver: \[[\'\"]?{}[\'\"]?, ''\]".format(current_version.replace(".", r"\.")),
+                "nxf_ver: ['{}', '']".format(new_version),
             )
         ],
     )

nf_core/download.py

@@ -605,8 +605,6 @@ def singularity_image_filenames(self, container):
             out_name = out_name[:-4]
         # Strip : and / characters
         out_name = out_name.replace("/", "-").replace(":", "-")
-        # Stupid Docker Hub not allowing hyphens
-        out_name = out_name.replace("nfcore", "nf-core")
         # Add file extension
         out_name = out_name + extension
 

nf_core/lint/__init__.py

@@ -113,6 +113,7 @@ class PipelineLint(nf_core.utils.Pipeline):
     from .readme import readme
     from .schema_lint import schema_lint
     from .schema_params import schema_params
+    from .schema_description import schema_description
     from .template_strings import template_strings
     from .version_consistency import version_consistency
 
@@ -147,6 +148,7 @@ def __init__(self, wf_path, release_mode=False, fix=(), key=(), fail_ignored=Fal
             "template_strings",
             "schema_lint",
             "schema_params",
+            "schema_description",
             "actions_schema_validation",
             "merge_markers",
         ]

nf_core/lint/schema_description.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+from logging import warn
+import nf_core.schema
+
+
+def schema_description(self):
+    """Check that every parameter in the schema has a description
+
+    The ``nextflow_schema.json`` pipeline schema should describe every flat parameter
+    Furthermore warns about parameters outside of groups
+
+    * Warning: Parameters in ``nextflow_schema.json`` without a description
+    * Warning: Parameters in ``nextflow_schema.json`` that are defined outside of a group
+    """
+    passed = []
+    warned = []
+    ignored = []
+
+    # First, get the top-level config options for the pipeline
+    # Schema object already created in the `schema_lint` test
+    self.schema_obj = nf_core.schema.PipelineSchema()
+    self.schema_obj.get_schema_path(self.wf_path)
+    self.schema_obj.get_wf_params()
+    self.schema_obj.no_prompts = True
+    self.schema_obj.load_lint_schema()
+
+    # Get parameters that should be ignored according to the linting config
+    ignore_params = self.lint_config.get("schema_description", [])
+
+    # Get ungrouped params
+    if "properties" in self.schema_obj.schema.keys():
+        ungrouped_params = self.schema_obj.schema["properties"].keys()
+        for up in ungrouped_params:
+            if up in ignore_params:
+                ignored.append(f"Ignored ungrouped param in schema: `{up}`")
+            else:
+                warned.append(f"Ungrouped param in schema: `{up}`")
+
+    # Iterate over groups and add warning for parameters without a description
+    for group_key in self.schema_obj.schema["definitions"].keys():
+        group = self.schema_obj.schema["definitions"][group_key]
+        for param_key, param in group["properties"].items():
+            if param_key in ignore_params:
+                ignored.append(f"Ignoring description check for param in schema: `{param_key}`")
+                continue
+            if "description" not in param.keys():
+                warned.append(f"No description provided in schema for parameter: `{param_key}`")
+
+    for ip in ignore_params:
+        ignored.append(f"Parameter is ignored: `{ip}`")
+
+    return {"passed": passed, "warned": warned, "ignored": ignored}

nf_core/lint/schema_params.py

@@ -17,10 +17,11 @@ def schema_params(self):
     failed = []
 
     # First, get the top-level config options for the pipeline
-    # Schema object already created in the `schema_lint` test
+    self.schema_obj = nf_core.schema.PipelineSchema()
     self.schema_obj.get_schema_path(self.wf_path)
     self.schema_obj.get_wf_params()
     self.schema_obj.no_prompts = True
+    self.schema_obj.load_lint_schema()
 
     # Remove any schema params not found in the config
     removed_params = self.schema_obj.remove_schema_notfound_configs()

nf_core/pipeline-template/.github/workflows/ci.yml

@@ -8,6 +8,9 @@ on:
   release:
     types: [published]
 
+# Uncomment if we need an edge release of Nextflow again
+# env: NXF_EDGE: 1
+
 jobs:
   test:
     name: Run workflow tests
@@ -20,7 +23,7 @@ jobs:
     strategy:
       matrix:
         # Nextflow versions: check pipeline minimum and current latest
-        nxf_ver: ['20.04.0', '21.03.0-edge']
+        nxf_ver: ['20.04.0', '']
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2

nf_core/pipeline-template/lib/NfcoreSchema.groovy

@@ -112,8 +112,14 @@ class NfcoreSchema {
             }
             // unexpected params
             def params_ignore = params.schema_ignore_params.split(',') + 'schema_ignore_params'
-            if (!expectedParams.contains(specifiedParam) && !params_ignore.contains(specifiedParam)) {
-                unexpectedParams.push(specifiedParam)
+            def expectedParamsLowerCase = expectedParams.collect{ it.replace("-", "").toLowerCase() }
+            def specifiedParamLowerCase = specifiedParam.replace("-", "").toLowerCase()
+            if (!expectedParams.contains(specifiedParam) && !params_ignore.contains(specifiedParam) && !expectedParamsLowerCase.contains(specifiedParamLowerCase)) {
+                // Temporarily remove camelCase/camel-case params #1035
+                def unexpectedParamsLowerCase = unexpectedParams.collect{ it.replace("-", "").toLowerCase()}
+                if (!unexpectedParamsLowerCase.contains(specifiedParamLowerCase)){
+                    unexpectedParams.push(specifiedParam)
+                }
             }
         }