Merge pull request #1168 from ErikDanielsson/modules-install-new-stru…

…cture

Adapt `modules install` to new directory structure

nf_core/modules/install.py

@@ -1,6 +1,4 @@
 import os
-import sys
-import json
 import questionary
 import logging
 
@@ -49,10 +47,9 @@ def install(self, module):
             log.error("Module '{}' not found in list of available modules.".format(module))
             log.info("Use the command 'nf-core modules list' to view available software")
             return False
+
         # Set the install folder based on the repository name
-        install_folder = ["nf-core", "software"]
-        if not self.modules_repo.name == "nf-core/modules":
-            install_folder = ["external"]
+        install_folder = [self.modules_repo.user, self.modules_repo.repo]
 
         # Compute the module directory
         module_dir = os.path.join(self.dir, "modules", *install_folder, module)
@@ -62,7 +59,7 @@ def install(self, module):
         if not modules_json:
             return False
 
-        current_entry = modules_json["modules"].get(module)
+        current_entry = modules_json["repos"].get(module)
 
         if current_entry is not None and self.sha is None:
             # Fetch the latest commit for the module
@@ -130,7 +127,7 @@ def install(self, module):
             return False
 
         # Update module.json with newly installed module
-        self.update_modules_json(modules_json, module, version)
+        self.update_modules_json(modules_json, self.modules_repo.name, module, version)
         return True
 
     def check_module_files_installed(self, module_name, module_dir):

nf_core/modules/module_utils.py

@@ -25,9 +25,11 @@ class ModuleException(Exception):
     pass
 
 
-def get_module_git_log(module_name, per_page=30, page_nbr=1, since="2020-11-25T00:00:00Z"):
+def get_module_git_log(module_name, per_page=30, page_nbr=1, since="2021-07-07T00:00:00Z"):
     """
-    Fetches the commit history the of requested module
+    Fetches the commit history the of requested module since a given date. The default value is
+    not arbitrary - it is the last time the structure of the nf-core/modules repository was had an
+    update breaking backwards compatibility.
     Args:
         module_name (str): Name of module
         per_page (int): Number of commits per page returned by API
@@ -38,7 +40,7 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1, since="2020-11-25T0
     Returns:
         [ dict ]: List of commit SHAs and associated (truncated) message
     """
-    api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}&per_page={per_page}&page={page_nbr}&since={since}"
+    api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=modules/{module_name}&per_page={per_page}&page={page_nbr}&since={since}"
     log.debug(f"Fetching commit history of module '{module_name}' from github API")
     response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
     if response.status_code == 200:
@@ -102,13 +104,28 @@ def create_modules_json(pipeline_dir):
     pipeline_config = nf_core.utils.fetch_wf_config(pipeline_dir)
     pipeline_name = pipeline_config["manifest.name"]
     pipeline_url = pipeline_config["manifest.homePage"]
-    modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "modules": {}}
-    all_module_file_paths = glob.glob(f"{pipeline_dir}/modules/nf-core/software/**/*", recursive=True)
+    modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": dict()}
+    modules_dir = f"{pipeline_dir}/modules"
+
+    # Extract all modules repos in the pipeline directory
+    repo_names = [
+        f"{user_name}/{repo_name}"
+        for user_name in os.listdir(modules_dir)
+        if os.path.isdir(os.path.join(modules_dir, user_name)) and user_name != "local"
+        for repo_name in os.listdir(os.path.join(modules_dir, user_name))
+    ]
 
-    # Extract the module paths from the file paths
-    module_paths = list(set(map(os.path.dirname, filter(os.path.isfile, all_module_file_paths))))
-    module_names = [path.replace(f"{pipeline_dir}/modules/nf-core/software/", "") for path in module_paths]
-    module_repo = ModulesRepo()
+    # Get all module names in the repos
+    repo_module_names = {
+        repo_name: list(
+            {
+                os.path.relpath(os.path.dirname(path), os.path.join(modules_dir, repo_name))
+                for path in glob.glob(f"{modules_dir}/{repo_name}/**/*", recursive=True)
+                if os.path.isfile(path)
+            }
+        )
+        for repo_name in repo_names
+    }
 
     progress_bar = rich.progress.Progress(
         "[bold blue]{task.description}",
@@ -118,28 +135,33 @@ def create_modules_json(pipeline_dir):
     )
     with progress_bar:
         file_progress = progress_bar.add_task(
-            "Creating 'modules.json' file", total=len(module_names), test_name="module.json"
+            "Creating 'modules.json' file", total=sum(map(len, repo_module_names.values())), test_name="module.json"
         )
-        for module_name, module_path in zip(module_names, module_paths):
-            progress_bar.update(file_progress, advance=1, test_name=module_name)
-            try:
-                # Find the correct commit SHA for the local files.
-                # We iterate over the commit log pages until we either
-                # find a matching commit or we reach the end of the commits
-                correct_commit_sha = None
-                commit_page_nbr = 1
-                while correct_commit_sha is None:
-
-                    commit_shas = [
-                        commit["git_sha"] for commit in get_module_git_log(module_name, page_nbr=commit_page_nbr)
-                    ]
-                    correct_commit_sha = find_correct_commit_sha(module_name, module_path, module_repo, commit_shas)
-                    commit_page_nbr += 1
-
-                modules_json["modules"][module_name] = {"git_sha": correct_commit_sha}
-            except LookupError as e:
-                log.error(e)
-                raise UserWarning("Will not create 'modules.json' file")
+        for repo_name, module_names in repo_module_names.items():
+            module_repo = ModulesRepo(repo=repo_name)
+            repo_path = os.path.join(modules_dir, repo_name)
+            modules_json["repos"][repo_name] = dict()
+            for module_name in module_names:
+                module_path = os.path.join(repo_path, module_name)
+                progress_bar.update(file_progress, advance=1, test_name=f"{repo_name}/{module_name}")
+                try:
+                    # Find the correct commit SHA for the local files.
+                    # We iterate over the commit log pages until we either
+                    # find a matching commit or we reach the end of the commits
+                    correct_commit_sha = None
+                    commit_page_nbr = 1
+                    while correct_commit_sha is None:
+
+                        commit_shas = [
+                            commit["git_sha"] for commit in get_module_git_log(module_name, page_nbr=commit_page_nbr)
+                        ]
+                        correct_commit_sha = find_correct_commit_sha(module_name, module_path, module_repo, commit_shas)
+                        commit_page_nbr += 1
+
+                    modules_json["repos"][repo_name][module_name] = {"git_sha": correct_commit_sha}
+                except LookupError as e:
+                    log.error(e)
+                    raise UserWarning("Will not create 'modules.json' file")
     modules_json_path = os.path.join(pipeline_dir, "modules.json")
     with open(modules_json_path, "w") as fh:
         json.dump(modules_json, fh, indent=4)
@@ -187,7 +209,7 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_
     files_are_equal = [False, False, False]
     remote_copies = [None, None, None]
 
-    module_base_url = f"https://raw.githubusercontent.com/{modules_repo.name}/{commit_sha}/software/{module_name}"
+    module_base_url = f"https://raw.githubusercontent.com/{modules_repo.name}/{commit_sha}/modules/{module_name}"
     for i, file in enumerate(files_to_check):
         # Download remote copy and compare
         api_url = f"{module_base_url}/{file}"
@@ -255,7 +277,7 @@ def get_installed_modules(dir, repo_type="modules"):
     local_modules = []
     nfcore_modules = []
     local_modules_dir = None
-    nfcore_modules_dir = os.path.join(dir, "modules", "nf-core", "software")
+    nfcore_modules_dir = os.path.join(dir, "modules", "nf-core", "modules")
 
     # Get local modules
     if repo_type == "pipeline":
@@ -268,7 +290,7 @@ def get_installed_modules(dir, repo_type="modules"):
 
     # nf-core/modules
     if repo_type == "modules":
-        nfcore_modules_dir = os.path.join(dir, "software")
+        nfcore_modules_dir = os.path.join(dir, "modules")
 
     # Get nf-core modules
     if os.path.exists(nfcore_modules_dir):
@@ -306,7 +328,7 @@ def get_repo_type(dir):
     # Determine repository type
     if os.path.exists(os.path.join(dir, "main.nf")):
         return "pipeline"
-    elif os.path.exists(os.path.join(dir, "software")):
+    elif os.path.exists(os.path.join(dir, "modules")):
         return "modules"
     else:
         raise LookupError("Could not determine repository type of '{}'".format(dir))

nf_core/modules/modules_command.py

@@ -1,3 +1,4 @@
+from nf_core import modules
 import os
 import glob
 import shutil
@@ -35,9 +36,9 @@ def get_pipeline_modules(self):
         """Get list of modules installed in the current directory"""
         self.module_names = []
         if self.repo_type == "pipeline":
-            module_base_path = f"{self.dir}/modules/nf-core/software"
+            module_base_path = f"{self.dir}/modules/nf-core/modules"
         elif self.repo_type == "modules":
-            module_base_path = f"{self.dir}/software"
+            module_base_path = f"{self.dir}/modules"
         else:
             log.error("Directory is neither a clone of nf-core/modules nor a pipeline")
             raise SystemError
@@ -118,16 +119,18 @@ def load_modules_json(self):
             modules_json = None
         return modules_json
 
+    def update_modules_json(self, modules_json, repo_name, module_name, module_version):
+        """Updates the 'module.json' file with new module info"""
+        if repo_name not in modules_json["repos"]:
+            modules_json["repos"][repo_name] = dict()
+        modules_json["repos"][repo_name][module_name] = {"git_sha": module_version}
+        self.dump_modules_json(modules_json)
+
     def dump_modules_json(self, modules_json):
         modules_json_path = os.path.join(self.dir, "modules.json")
         with open(modules_json_path, "w") as fh:
             json.dump(modules_json, fh, indent=4)
 
-    def update_modules_json(self, modules_json, module_name, module_version):
-        """Updates the 'module.json' file with new module info"""
-        modules_json["modules"][module_name] = {"git_sha": module_version}
-        self.dump_modules_json(modules_json)
-
     def load_lint_config(self):
         """Parse a pipeline lint config file.
 

nf_core/modules/modules_repo.py

@@ -18,6 +18,7 @@ class ModulesRepo(object):
 
     def __init__(self, repo="nf-core/modules", branch="master"):
         self.name = repo
+        self.user, self.repo = self.name.split("/")
         self.branch = branch
         self.modules_file_tree = {}
         self.modules_current_hash = None
@@ -46,9 +47,9 @@ def get_modules_file_tree(self):
         self.modules_current_hash = result["sha"]
         self.modules_file_tree = result["tree"]
         for f in result["tree"]:
-            if f["path"].startswith("software/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]:
-                # remove software/ and /main.nf
-                self.modules_avail_module_names.append(f["path"][9:-8])
+            if f["path"].startswith(f"modules/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]:
+                # remove modules/ and /main.nf
+                self.modules_avail_module_names.append(f["path"].replace("modules/", "").replace("/main.nf", ""))
 
     def get_module_file_urls(self, module, commit=""):
         """Fetch list of URLs for a specific module
@@ -73,7 +74,7 @@ def get_module_file_urls(self, module, commit=""):
         """
         results = {}
         for f in self.modules_file_tree:
-            if not f["path"].startswith("software/{}".format(module)):
+            if not f["path"].startswith("modules/{}".format(module)):
                 continue
             if f["type"] != "blob":
                 continue

tests/modules/install.py

@@ -25,14 +25,14 @@ def test_modules_install_nomodule(self):
 def test_modules_install_trimgalore(self):
     """Test installing a module - TrimGalore!"""
     assert self.mods_install.install("trimgalore") is not False
-    module_path = os.path.join(self.mods_install.dir, "modules", "nf-core", "software", "trimgalore")
+    module_path = os.path.join(self.mods_install.dir, "modules", "nf-core", "modules", "trimgalore")
     assert os.path.exists(module_path)
 
 
 def test_modules_install_trimgalore_alternative_source(self):
     """Test installing a module from a different source repository - TrimGalore!"""
     assert self.mods_install_alt.install("trimgalore") is not False
-    module_path = os.path.join(self.mods_install.dir, "modules", "external", "trimgalore")
+    module_path = os.path.join(self.mods_install.dir, "modules", "ewels", "nf-core-modules", "trimgalore")
     assert os.path.exists(module_path)