diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 8a45922c7c..34feec23f1 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -27,7 +27,7 @@ jobs: - name: Sync dev docs # Only sync with the website if it was a push from nf-core/tools dev branch - if: github.repository == 'nf-core/tools' && github.event.type == 'push' && github.event.base_ref == 'refs/heads/dev' + if: github.repository == 'nf-core/tools' && github.event_name == 'push' && github.event.ref == 'refs/heads/dev' uses: SamKirkland/FTP-Deploy-Action@4.0.0 with: server: ${{ secrets.ftp_server }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 19596e5c4f..6a63afa13b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### General * Convert nf-core/tools API / lint test documentation to MyST ([#1245](https://github.com/nf-core/tools/pull/1245)) +* Build documentation for the `nf-core modules lint` tests ([#1250](https://github.com/nf-core/tools/pull/1250)) ### Modules diff --git a/docs/api/_src/api/lint.md b/docs/api/_src/api/lint.md index a0e3a6d5c9..cb75b0ecb5 100644 --- a/docs/api/_src/api/lint.md +++ b/docs/api/_src/api/lint.md @@ -1,7 +1,7 @@ # nf_core.lint :::{seealso} -See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. +See the [Lint Tests](../pipeline_lint_tests/index.md) docs for information about specific linting functions. ::: ```{eval-rst} diff --git a/docs/api/_src/index.md b/docs/api/_src/index.md index a6fb767122..de39e7244b 100644 --- a/docs/api/_src/index.md +++ b/docs/api/_src/index.md @@ -13,12 +13,10 @@ api/index.rst This documentation is for the `nf-core/tools` package. -Primarily, it describes the different [code lint tests](lint_tests/index.html) -run by `nf-core lint` (typically visited by a developer when their pipeline fails a given -test), and also reference for the `nf_core` [Python package API](api/index.html). - -## Indices and tables +## Contents +- [Pipeline code lint tests](pipeline_lint_tests/index.md) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index.md) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index.md) - {ref}`genindex` - {ref}`modindex` -- {ref}`search` diff --git a/docs/api/_src/module_lint_tests/index.md b/docs/api/_src/module_lint_tests/index.md new file mode 100644 index 0000000000..f889abf73c --- /dev/null +++ b/docs/api/_src/module_lint_tests/index.md @@ -0,0 +1,9 @@ +# Module lint tests + +```{toctree} +:caption: 'Tests:' +:glob: true +:maxdepth: 2 + +* +``` diff --git a/docs/api/_src/module_lint_tests/main_nf.md b/docs/api/_src/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..5a96785733 --- /dev/null +++ b/docs/api/_src/module_lint_tests/main_nf.md @@ -0,0 +1,5 @@ +# main_nf + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.main_nf +``` diff --git a/docs/api/_src/module_lint_tests/meta_yml.md b/docs/api/_src/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..33920a4b9b --- /dev/null +++ b/docs/api/_src/module_lint_tests/meta_yml.md @@ -0,0 +1,5 @@ +# meta_yml + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.meta_yml +``` diff --git a/docs/api/_src/module_lint_tests/module_deprecations.md b/docs/api/_src/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..f00b4dc54d --- /dev/null +++ b/docs/api/_src/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.module_deprecations +``` diff --git a/docs/api/_src/module_lint_tests/module_todos.md b/docs/api/_src/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..d592addd06 --- /dev/null +++ b/docs/api/_src/module_lint_tests/module_todos.md @@ -0,0 +1,5 @@ +# module_todos + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.module_todos +``` diff --git a/docs/api/_src/pipeline_lint_tests/index.md b/docs/api/_src/pipeline_lint_tests/index.md index d10f8421c5..0cf9bc1d21 100644 --- a/docs/api/_src/pipeline_lint_tests/index.md +++ b/docs/api/_src/pipeline_lint_tests/index.md @@ -1,4 +1,4 @@ -# Pipline lint tests +# Pipeline lint tests ```{toctree} :caption: 'Tests:' diff --git a/docs/api/make_lint_md.py b/docs/api/make_lint_md.py new file mode 100644 index 0000000000..a6ec98a944 --- /dev/null +++ b/docs/api/make_lint_md.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import fnmatch +import os +import nf_core.lint +import nf_core.modules.lint + + +def make_docs(docs_basedir, lint_tests, md_template): + # Get list of existing .md files + existing_docs = [] + for fn in os.listdir(docs_basedir): + if fnmatch.fnmatch(fn, "*.md") and not fnmatch.fnmatch(fn, "index.md"): + existing_docs.append(os.path.join(docs_basedir, fn)) + + for test_name in lint_tests: + fn = os.path.join(docs_basedir, "{}.md".format(test_name)) + if os.path.exists(fn): + existing_docs.remove(fn) + else: + with open(fn, "w") as fh: + fh.write(md_template.format(test_name)) + print(test_name) + + for fn in existing_docs: + os.remove(fn) + + +# Create the pipeline docs +pipeline_docs_basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_src", "pipeline_lint_tests") +make_docs( + pipeline_docs_basedir, + nf_core.lint.PipelineLint._get_all_lint_tests(True), + """# {0} + +```{{eval-rst}} +.. automethod:: nf_core.lint.PipelineLint.{0} +``` +""", +) + +# Create the modules lint docs +modules_docs_basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_src", "module_lint_tests") +make_docs( + modules_docs_basedir, + nf_core.modules.lint.ModuleLint._get_all_lint_tests(), + """# {0} + +```{{eval-rst}} +.. automethod:: nf_core.modules.lint.ModuleLint.{0} +``` +""", +) diff --git a/docs/api/make_lint_rst.py b/docs/api/make_lint_rst.py deleted file mode 100644 index 2e80e384ac..0000000000 --- a/docs/api/make_lint_rst.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -import fnmatch -import os -import nf_core.lint - -docs_basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_src", "pipeline_lint_tests") - -# Get list of existing .md files -existing_docs = [] -for fn in os.listdir(docs_basedir): - if fnmatch.fnmatch(fn, "*.md") and not fnmatch.fnmatch(fn, "index.md"): - existing_docs.append(os.path.join(docs_basedir, fn)) - -# Make .md file for each test name -lint_obj = nf_core.lint.PipelineLint("", True) -md_template = """# {0} - -```{eval-rst} -.. automethod:: nf_core.lint.PipelineLint.{0} -``` - -""" - -for test_name in lint_obj.lint_tests: - fn = os.path.join(docs_basedir, "{}.md".format(test_name)) - if os.path.exists(fn): - existing_docs.remove(fn) - else: - with open(fn, "w") as fh: - fh.write(md_template.format(test_name)) - -for fn in existing_docs: - os.remove(fn) diff --git a/nf_core/lint/modules_json.py b/nf_core/lint/modules_json.py index 07abf94987..4c5923d508 100644 --- a/nf_core/lint/modules_json.py +++ b/nf_core/lint/modules_json.py @@ -8,7 +8,7 @@ def modules_json(self): """Make sure all modules described in the ``modules.json`` file are actually installed Every module installed from ``nf-core/modules`` must have an entry in the ``modules.json`` file - with an associated version git_sha hash. + with an associated version commit hash. * Failure: If module entries are found in ``modules.json`` for modules that are not installed """ diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 80fe8fbf9e..abc9e48dc1 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -9,11 +9,24 @@ def main_nf(module_lint_object, module): """ - Lint a single main.nf module file + Lint a ``main.nf`` module file + Can also be used to lint local module files, - in which case failures should be interpreted - as warnings + in which case failures will be reported as + warnings. + + The test checks for the following: + + * Software versions and containers are valid + * The module has a process label and it is among + the standard ones. + * If a ``meta`` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the ``saveAs`` function. + * The module script section should contain definitions + of ``software`` and ``prefix`` """ + inputs = [] outputs = [] @@ -112,7 +125,7 @@ def main_nf(module_lint_object, module): def check_script_section(self, lines): """ Lint the script section - Checks whether 'def prefix' is defined and whether getProcessName is used for `versions.yml`. + Checks whether `def prefix` is defined and whether getProcessName is used for `versions.yml`. """ script = "".join(lines) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index be265a16ec..3c59c774d4 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -7,7 +7,19 @@ def meta_yml(module_lint_object, module): - """Lint a meta yml file""" + """ + Lint a ``meta.yml`` file + + The lint test checks that the module has + a ``meta.yml`` file and that it contains + the required keys: ``name``, input`` and + ``output``. + + In addition it checks that the module name + and module input is consistent between the + ``meta.yml`` and the ``main.nf``. + + """ required_keys = ["name", "input", "output"] required_keys_lists = ["input", "output"] try: diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index 4e466a2c3c..b77b54c3f8 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -11,10 +11,12 @@ def module_changes(module_lint_object, module): """ Checks whether installed nf-core modules have changed compared to the original repository - Downloads the 'main.nf' and 'meta.yml' files for every module + + Downloads the ``main.nf`` and ``meta.yml`` files for every module and compares them to the local copies - If the module has a 'git_sha', the file content is checked against this sha + If the module has a commit SHA entry in the ``modules.json``, the file content is + compared against the files in the remote at this SHA. Only runs when linting a pipeline, not the modules repository """ diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index a8470d0d18..b616daa37f 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -3,7 +3,6 @@ """ import os import logging -import sys import yaml log = logging.getLogger(__name__) @@ -11,7 +10,13 @@ def module_tests(module_lint_object, module): """ - Lint module tests + Lint the tests of a module in ``nf-core/modules`` + + It verifies that the test directory exists + and contains a ``main.nf`` and a ``test.yml``, + and that the module is present in the ``pytest_modules.yml`` + file. + """ if os.path.exists(module.test_dir): diff --git a/nf_core/modules/lint/module_todos.py b/nf_core/modules/lint/module_todos.py index 94b10299a8..33b35415b6 100644 --- a/nf_core/modules/lint/module_todos.py +++ b/nf_core/modules/lint/module_todos.py @@ -8,8 +8,28 @@ def module_todos(module_lint_object, module): """ Look for TODO statements in the module files - Slight modification of the "nf_core.lint.pipeline_todos" function to make it work - for a single module + + The nf-core module template contains a number of comment lines to help developers + of new modules know where they need to edit files and add content. + They typically have the following format: + + .. code-block:: groovy + + // TODO nf-core: Make some kind of change to the workflow here + + ..or in markdown: + + .. code-block:: html + + + + This lint test runs through all files in the module and searches for these lines. + If any are found they will throw a warning. + + .. tip:: Note that many GUI code editors have plugins to list all instances of *TODO* + in a given project directory. This is a very quick and convenient way to get + started on your pipeline! + """ # Main module directory diff --git a/nf_core/modules/lint/module_version.py b/nf_core/modules/lint/module_version.py index cc3e01c8be..29d7327490 100644 --- a/nf_core/modules/lint/module_version.py +++ b/nf_core/modules/lint/module_version.py @@ -18,9 +18,13 @@ def module_version(module_lint_object, module): """ - Verify that the module has a version (git_sha) specified in the - modules.json file and checks whether a new version is available + Verifies that the module has a version specified in the ``modules.json`` file + + It checks whether the module has an entry in the ``modules.json`` file + containing a commit SHA. If that is true, it verifies that there are no + newer version of the module available. """ + modules_json_path = os.path.join(module_lint_object.dir, "modules.json") # Verify that a git_sha exists in the `modules.json` file for this module