Merge pull request #1641 from ErikDanielsson/get-generic-git

Add support for generic git remotes

CHANGELOG.md

@@ -24,6 +24,7 @@
 
 - Add `--fix-version` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588))
 - Fix a bug in the regex extracting the version from biocontainers URLs ([#1598](https://github.com/nf-core/tools/pull/1598))
+- Update how we interface with git remotes. ([#1626](https://github.com/nf-core/tools/issues/1626))
 
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 

README.md

@@ -911,38 +911,19 @@ The nf-core DSL2 modules repository is at <https://github.com/nf-core/modules>
 
 The modules supercommand comes with two flags for specifying a custom remote:
 
-- `--github-repository <github repo>`: Specify the repository from which the modules should be fetched. Defaults to `nf-core/modules`.
-- `--branch <branch name>`: Specify the branch from which the modules shoudl be fetched. Defaults to `master`.
+- `--git-remote <git remote url>`: Specify the repository from which the modules should be fetched as a git URL. Defaults to the github repository of `nf-core/modules`.
+- `--branch <branch name>`: Specify the branch from which the modules should be fetched. Defaults to the default branch of your repository.
 
 Note that a custom remote must follow a similar directory structure to that of `nf-core/moduleś` for the `nf-core modules` commands to work properly.
 
-### Private remote modules
+The modules commands will during initalisation try to pull changes from the remote repositories. If you want to disable this, for example
+due to performance reason or if you want to run the commands offline, you can use the flag `--no-pull`. Note however that the commands will
+still need to clone repositories that have previously not been used.
 
-In order to get access to your private modules repo, you need to create
-the `~/.config/gh/hosts.yml` file, which is the same file required by
-[GitHub CLI](https://cli.github.com/) to deal with private repositories.
-Such file is structured as follow:
+### Private remote repositories
 
-```conf
-github.com:
-    oauth_token: <your github access token>
-    user: <your github user>
-    git_protocol: <ssh or https are valid choices>
-```
-
-The easiest way to create this configuration file is through _GitHub CLI_: follow
-its [installation instructions](https://cli.github.com/manual/installation)
-and then call:
-
-```bash
-gh auth login
-```
-
-After that, you will be able to list and install your private modules without
-providing your github credentials through command line, by using `--github-repository`
-and `--branch` options properly.
-See the documentation on [gh auth login](https://cli.github.com/manual/gh_auth_login>)
-to get more information.
+You can use the modules command with private remote repositories. Make sure that your local `git` is correctly configured with your private remote
+and then specify the remote the same way you would do with a public remote repository.
 
 ### List modules
 
@@ -1079,7 +1060,7 @@ There are three additional flags that you can use when installing a module:
 
 - `--force`: Overwrite a previously installed version of the module.
 - `--prompt`: Select the module version using a cli prompt.
-- `--sha <commit_sha>`: Install the module at a specific commit from the `nf-core/modules` repository.
+- `--sha <commit_sha>`: Install the module at a specific commit.
 
 ### Update modules in a pipeline
 

nf_core/__main__.py

@@ -28,6 +28,9 @@
 # Submodules should all traverse back to this
 log = logging.getLogger()
 
+# Set up .nfcore directory for storing files between sessions
+nf_core.utils.setup_nfcore_dir()
+
 # Set up nicer formatting of click cli help messages
 click.rich_click.MAX_WIDTH = 100
 click.rich_click.USE_RICH_MARKUP = True
@@ -342,27 +345,31 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdow
 @nf_core_cli.group()
 @click.option(
     "-g",
-    "--github-repository",
+    "--git-remote",
     type=str,
-    default="nf-core/modules",
-    help="GitHub repository hosting modules.",
+    default=nf_core.modules.modules_repo.NF_CORE_MODULES_REMOTE,
+    help="Remote git repo to fetch files from",
+)
+@click.option("-b", "--branch", type=str, default=None, help="Branch of git repository hosting modules.")
+@click.option(
+    "--no-pull",
+    is_flag=True,
+    default=False,
+    help="Do not pull in latest changes to local clone of modules repository.",
 )
-@click.option("-b", "--branch", type=str, default="master", help="Branch of GitHub repository hosting modules.")
 @click.pass_context
-def modules(ctx, github_repository, branch):
+def modules(ctx, git_remote, branch, no_pull):
     """
     Commands to manage Nextflow DSL2 modules (tool wrappers).
     """
     # ensure that ctx.obj exists and is a dict (in case `cli()` is called
     # by means other than the `if` block below)
     ctx.ensure_object(dict)
 
-    # Make repository object to pass to subcommands
-    try:
-        ctx.obj["modules_repo_obj"] = nf_core.modules.ModulesRepo(github_repository, branch)
-    except LookupError as e:
-        log.critical(e)
-        sys.exit(1)
+    # Place the arguments in a context object
+    ctx.obj["modules_repo_url"] = git_remote
+    ctx.obj["modules_repo_branch"] = branch
+    ctx.obj["modules_repo_no_pull"] = no_pull
 
 
 # nf-core modules list subcommands
@@ -385,10 +392,11 @@ def remote(ctx, keywords, json):
     List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
     try:
-        module_list = nf_core.modules.ModuleList(None, remote=True)
-        module_list.modules_repo = ctx.obj["modules_repo_obj"]
+        module_list = nf_core.modules.ModuleList(
+            None, True, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         print(module_list.list_modules(keywords, json))
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
 
@@ -410,10 +418,11 @@ def local(ctx, keywords, json, dir):
     List modules installed locally in a pipeline
     """
     try:
-        module_list = nf_core.modules.ModuleList(dir, remote=False)
-        module_list.modules_repo = ctx.obj["modules_repo_obj"]
+        module_list = nf_core.modules.ModuleList(
+            dir, False, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         print(module_list.list_modules(keywords, json))
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
 
@@ -439,12 +448,19 @@ def install(ctx, tool, dir, prompt, force, sha):
     Fetches and installs module files from a remote repo e.g. nf-core/modules.
     """
     try:
-        module_install = nf_core.modules.ModuleInstall(dir, force=force, prompt=prompt, sha=sha)
-        module_install.modules_repo = ctx.obj["modules_repo_obj"]
+        module_install = nf_core.modules.ModuleInstall(
+            dir,
+            force,
+            prompt,
+            sha,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+        )
         exit_status = module_install.install(tool)
         if not exit_status and all:
             sys.exit(1)
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.error(e)
         sys.exit(1)
 
@@ -487,13 +503,21 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff):
     """
     try:
         module_install = nf_core.modules.ModuleUpdate(
-            dir, force=force, prompt=prompt, sha=sha, update_all=all, show_diff=preview, save_diff_fn=save_diff
+            dir,
+            force,
+            prompt,
+            sha,
+            all,
+            preview,
+            save_diff,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
         )
-        module_install.modules_repo = ctx.obj["modules_repo_obj"]
         exit_status = module_install.update(tool)
         if not exit_status and all:
             sys.exit(1)
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.error(e)
         sys.exit(1)
 
@@ -514,10 +538,11 @@ def remove(ctx, dir, tool):
     Remove a module from a pipeline.
     """
     try:
-        module_remove = nf_core.modules.ModuleRemove(dir)
-        module_remove.modules_repo = ctx.obj["modules_repo_obj"]
+        module_remove = nf_core.modules.ModuleRemove(
+            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         module_remove.remove(tool)
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
 
@@ -562,6 +587,9 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
+    except LookupError as e:
+        log.error(e)
+        sys.exit(1)
 
 
 # nf-core modules create-test-yml
@@ -582,7 +610,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
     try:
         meta_builder = nf_core.modules.ModulesTestYmlBuilder(tool, run_tests, output, force, no_prompts)
         meta_builder.run()
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
 
@@ -608,8 +636,9 @@ def lint(ctx, tool, dir, key, all, local, passed, fix_version):
     nf-core/modules repository.
     """
     try:
-        module_lint = nf_core.modules.ModuleLint(dir=dir)
-        module_lint.modules_repo = ctx.obj["modules_repo_obj"]
+        module_lint = nf_core.modules.ModuleLint(
+            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         module_lint.lint(
             module=tool,
             key=key,
@@ -624,7 +653,7 @@ def lint(ctx, tool, dir, key, all, local, passed, fix_version):
     except nf_core.modules.lint.ModuleLintException as e:
         log.error(e)
         sys.exit(1)
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
 
@@ -653,10 +682,11 @@ def info(ctx, tool, dir):
     If not, usage from the remote modules repo will be shown.
     """
     try:
-        module_info = nf_core.modules.ModuleInfo(dir, tool)
-        module_info.modules_repo = ctx.obj["modules_repo_obj"]
+        module_info = nf_core.modules.ModuleInfo(
+            dir, tool, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         print(module_info.get_module_info())
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.error(e)
         sys.exit(1)
 
@@ -674,12 +704,14 @@ def bump_versions(ctx, tool, dir, all, show_all):
     the nf-core/modules repo.
     """
     try:
-        version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=dir)
+        version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(
+            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+        )
         version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all)
     except nf_core.modules.module_utils.ModuleException as e:
         log.error(e)
         sys.exit(1)
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
 
@@ -742,7 +774,7 @@ def test_module(ctx, tool, no_prompts, pytest_args):
     try:
         meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args)
         meta_builder.run()
-    except UserWarning as e:
+    except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
 

nf_core/lint/modules_json.py

@@ -27,7 +27,14 @@ def modules_json(self):
         all_modules_passed = True
 
         for repo in modules_json["repos"].keys():
-            for key in modules_json["repos"][repo].keys():
+            # Check if the modules.json has been updated to keep the
+            if "modules" not in modules_json["repos"][repo] or "git_url" not in modules_json["repos"][repo]:
+                failed.append(
+                    f"Your `modules.json` file is outdated. Please remove it and reinstall it by running any module command"
+                )
+                continue
+
+            for key in modules_json["repos"][repo]["modules"]:
                 if not key in modules_command.module_names[repo]:
                     failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.")
                     all_modules_passed = False

nf_core/modules/bump_versions.py

@@ -28,8 +28,8 @@
 
 
 class ModuleVersionBumper(ModuleCommand):
-    def __init__(self, pipeline_dir):
-        super().__init__(pipeline_dir)
+    def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False):
+        super().__init__(pipeline_dir, remote_url, branch, no_pull)
 
         self.up_to_date = None
         self.updated = None

nf_core/modules/info.py

@@ -11,20 +11,15 @@
 from rich.table import Table
 from rich.text import Text
 
-from .module_utils import (
-    get_installed_modules,
-    get_module_git_log,
-    get_repo_type,
-    module_exist_in_repo,
-)
+from .module_utils import get_repo_type
 from .modules_command import ModuleCommand
 from .modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)
 
 
 class ModuleInfo(ModuleCommand):
-    def __init__(self, pipeline_dir, tool):
+    def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull):
 
         self.module = tool
         self.meta = None
@@ -40,7 +35,7 @@ def __init__(self, pipeline_dir, tool):
                 log.debug(f"Only showing remote info: {e}")
                 pipeline_dir = None
 
-        super().__init__(pipeline_dir)
+        super().__init__(pipeline_dir, remote_url, branch, no_pull)
 
     def get_module_info(self):
         """Given the name of a module, parse meta.yml and print usage help."""
@@ -93,28 +88,14 @@ def get_remote_yaml(self):
         Returns:
             dict or bool: Parsed meta.yml found, False otherwise
         """
-        # Fetch the remote repo information
-        self.modules_repo.get_modules_file_tree()
-
         # Check if our requested module is there
-        if self.module not in self.modules_repo.modules_avail_module_names:
+        if self.module not in self.modules_repo.get_avail_modules():
             return False
 
-        # Get the remote path
-        meta_url = None
-        for file_dict in self.modules_repo.modules_file_tree:
-            if file_dict.get("path") == f"modules/{self.module}/meta.yml":
-                meta_url = file_dict.get("url")
-
-        if not meta_url:
+        file_contents = self.modules_repo.get_meta_yml(self.module)
+        if file_contents is None:
             return False
-
-        # Download and parse
-        log.debug(f"Attempting to fetch {meta_url}")
-        response = requests.get(meta_url)
-        result = response.json()
-        file_contents = base64.b64decode(result["content"])
-        self.remote_location = self.modules_repo.name
+        self.remote_location = self.modules_repo.fullname
         return yaml.safe_load(file_contents)
 
     def generate_module_info_help(self):