From 4765d9afa3b5b6e561c2fed8d00ac8b9b30f8dea Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Wed, 16 Jun 2021 14:09:19 +0200
Subject: [PATCH 01/34] Added support for modules.json for modules install
 commandq

---
 nf_core/modules/pipeline_modules.py | 42 ++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 12f18bdb5..4d209dc5e 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -198,7 +198,8 @@ def list_modules(self, print_json=False):
     def install(self, module=None):
 
         # Check whether pipelines is valid
-        self.has_valid_pipeline()
+        if self.has_valid_pipeline():
+            self.has_modules_file()
 
         # Get the available modules
         self.modules_repo.get_modules_file_tree()
@@ -241,6 +242,15 @@ def install(self, module=None):
             self.modules_repo.download_gh_file(dl_filename, api_url)
         log.info("Downloaded {} files to {}".format(len(files), module_dir))
 
+        # Update module.json with new module
+        modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
+        with open(modules_json_path, "r") as fh:
+            modules_json = json.load(fh)
+        commit_sha = self.get_module_commit_sha(module)
+        modules_json["modules"][module] = {"git_sha": commit_sha}
+        with open(modules_json_path, "w") as fh:
+            json.dump(modules_json, fh, indent=4)
+
     def update(self, module, force=False):
         log.error("This command is not yet implemented")
         pass
@@ -317,3 +327,33 @@ def has_valid_pipeline(self):
         nf_config = os.path.join(self.pipeline_dir, "nextflow.config")
         if not os.path.exists(main_nf) and not os.path.exists(nf_config):
             raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.pipeline_dir}'")
+        return True
+
+    def has_modules_file(self):
+        """Checks whether a module.json file has been created and creates one if it is missing"""
+        modules_json = os.path.join(self.pipeline_dir, "modules.json")
+        if not os.path.exists(modules_json):
+            pipeline_config = nf_core.utils.fetch_wf_config(self.pipeline_dir)
+            pipeline_name = pipeline_config["manifest.name"]
+            pipeline_url = pipeline_config["manifest.homePage"]
+            modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "modules": {}}
+            module_names = [
+                path.replace(f"{self.pipeline_dir}/modules/nf-core/software/", "")
+                for path in glob.glob(f"{self.pipeline_dir}/modules/nf-core/software/*")
+            ]
+            for module_name in module_names:
+                commit_sha = self.get_module_commit_sha(module_name)
+                modules_json["modules"][module_name] = {"git_sha": commit_sha}
+            modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
+            with open(modules_json_path, "w") as fh:
+                json.dump(modules_json, fh, indent=4)
+
+    def get_module_commit_sha(self, module_name):
+        """Fetches the latests commit SHA for the requested module"""
+        api_url = f'https://api.github.com/repos/nf-core/modules/commits/master?q={{path="software/{module_name}"}}'
+        response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
+        if response.status_code == 200:
+            json_response = response.json()
+            return json_response["sha"]
+        else:
+            raise SystemError(f"Unable to fetch commit SHA for module {module_name}")

From bb62ddba73e03c445b1033347142c8fe932c28ec Mon Sep 17 00:00:00 2001
From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com>
Date: Wed, 16 Jun 2021 14:53:13 +0200
Subject: [PATCH 02/34] Remove redundant code

Co-authored-by: Kevin Menden <kevin.menden@live.com>
---
 nf_core/modules/pipeline_modules.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 4d209dc5e..cde0b9ad7 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -331,8 +331,8 @@ def has_valid_pipeline(self):
 
     def has_modules_file(self):
         """Checks whether a module.json file has been created and creates one if it is missing"""
-        modules_json = os.path.join(self.pipeline_dir, "modules.json")
-        if not os.path.exists(modules_json):
+        modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
+        if not os.path.exists(modules_json_path):
             pipeline_config = nf_core.utils.fetch_wf_config(self.pipeline_dir)
             pipeline_name = pipeline_config["manifest.name"]
             pipeline_url = pipeline_config["manifest.homePage"]
@@ -344,7 +344,6 @@ def has_modules_file(self):
             for module_name in module_names:
                 commit_sha = self.get_module_commit_sha(module_name)
                 modules_json["modules"][module_name] = {"git_sha": commit_sha}
-            modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
             with open(modules_json_path, "w") as fh:
                 json.dump(modules_json, fh, indent=4)
 

From 8af92af626684c6d8679cd28d140f3fa2827bb2e Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Wed, 16 Jun 2021 15:29:50 +0200
Subject: [PATCH 03/34] Implement suggestions from code review

---
 nf_core/modules/pipeline_modules.py | 40 +++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 4d209dc5e..173b6fe62 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -198,8 +198,7 @@ def list_modules(self, print_json=False):
     def install(self, module=None):
 
         # Check whether pipelines is valid
-        if self.has_valid_pipeline():
-            self.has_modules_file()
+        self.has_valid_pipeline()
 
         # Get the available modules
         self.modules_repo.get_modules_file_tree()
@@ -246,7 +245,28 @@ def install(self, module=None):
         modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
         with open(modules_json_path, "r") as fh:
             modules_json = json.load(fh)
-        commit_sha = self.get_module_commit_sha(module)
+        try:
+            commit_sha = self.get_module_commit_sha(module)
+        except SystemError as e:
+            log.error(e)
+            log.error(f"Will remove module '{module}'")
+            # Remove the module
+            try:
+                shutil.rmtree(module_dir)
+                # Try cleaning up empty parent if tool/subtool and tool/ is empty
+                if module.count("/") > 0:
+                    parent_dir = os.path.dirname(module_dir)
+                    try:
+                        os.rmdir(parent_dir)
+                    except OSError:
+                        log.debug(f"Parent directory not empty: '{parent_dir}'")
+                    else:
+                        log.debug(f"Deleted orphan tool directory: '{parent_dir}'")
+                return False
+            except OSError as e:
+                log.error("Could not remove module: {}".format(e))
+                return False
+
         modules_json["modules"][module] = {"git_sha": commit_sha}
         with open(modules_json_path, "w") as fh:
             json.dump(modules_json, fh, indent=4)
@@ -327,12 +347,14 @@ def has_valid_pipeline(self):
         nf_config = os.path.join(self.pipeline_dir, "nextflow.config")
         if not os.path.exists(main_nf) and not os.path.exists(nf_config):
             raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.pipeline_dir}'")
+        self.has_modules_file()
         return True
 
     def has_modules_file(self):
         """Checks whether a module.json file has been created and creates one if it is missing"""
         modules_json = os.path.join(self.pipeline_dir, "modules.json")
         if not os.path.exists(modules_json):
+            log.info("Creating missing 'module.json' file.")
             pipeline_config = nf_core.utils.fetch_wf_config(self.pipeline_dir)
             pipeline_name = pipeline_config["manifest.name"]
             pipeline_url = pipeline_config["manifest.homePage"]
@@ -342,8 +364,13 @@ def has_modules_file(self):
                 for path in glob.glob(f"{self.pipeline_dir}/modules/nf-core/software/*")
             ]
             for module_name in module_names:
-                commit_sha = self.get_module_commit_sha(module_name)
-                modules_json["modules"][module_name] = {"git_sha": commit_sha}
+                try:
+                    commit_sha = self.get_module_commit_sha(module_name)
+                    modules_json["modules"][module_name] = {"git_sha": commit_sha}
+                except SystemError as e:
+                    log.error(e)
+                    log.error("Will not create 'modules.json' file")
+                    sys.exit(1)
             modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
             with open(modules_json_path, "w") as fh:
                 json.dump(modules_json, fh, indent=4)
@@ -355,5 +382,8 @@ def get_module_commit_sha(self, module_name):
         if response.status_code == 200:
             json_response = response.json()
             return json_response["sha"]
+        elif response.status_code == 404:
+            log.error(f"Module '{module_name}' not found in 'nf-core/modules/'\n{api_url}")
+            sys.exit(1)
         else:
             raise SystemError(f"Unable to fetch commit SHA for module {module_name}")

From 33518410ee271f63e88cbd28ffe4ee8f416f5dde Mon Sep 17 00:00:00 2001
From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com>
Date: Wed, 16 Jun 2021 16:06:28 +0200
Subject: [PATCH 04/34] Apply changes from code review

Co-authored-by: Kevin Menden <kevin.menden@live.com>
---
 nf_core/modules/pipeline_modules.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 5b6af8eb6..fe1442217 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -354,7 +354,7 @@ def has_modules_file(self):
         """Checks whether a module.json file has been created and creates one if it is missing"""
         modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
         if not os.path.exists(modules_json_path):
-            log.info("Creating missing 'module.json' file.")
+            log.info("Creating missing 'modules.json' file.")
             pipeline_config = nf_core.utils.fetch_wf_config(self.pipeline_dir)
             pipeline_name = pipeline_config["manifest.name"]
             pipeline_url = pipeline_config["manifest.homePage"]

From 199216f9623d80285bda333dc26f80ca2e0fe55f Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Fri, 18 Jun 2021 15:37:56 +0200
Subject: [PATCH 05/34] Added API call for full git log

---
 nf_core/modules/module_utils.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 nf_core/modules/module_utils.py

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
new file mode 100644
index 000000000..07aeb1e00
--- /dev/null
+++ b/nf_core/modules/module_utils.py
@@ -0,0 +1,22 @@
+import requests
+import sys
+import logging
+import nf_core.utils
+
+log = logging.getLogger(__name__)
+
+
+def get_module_git_log(module_name):
+    """Fetches the git log_for the requested module"""
+    api_url = f'https://api.github.com/repos/nf-core/modules/commits?q={{sha=master, path="software/{module_name}"}}'
+    response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
+    if response.status_code == 200:
+        commits = [
+            {"git_sha": commit["sha"], "trunc_message": commit["commit"]["message"]} for commit in response.json()
+        ]
+        return commits
+    elif response.status_code == 404:
+        log.error(f"Module '{module_name}' not found in 'nf-core/modules/'\n{api_url}")
+        sys.exit(1)
+    else:
+        raise SystemError(f"Unable to fetch commit SHA for module {module_name}")

From ca5cb84fc4437ab25662f74e00f498808bd7e6d8 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Fri, 18 Jun 2021 17:30:08 +0200
Subject: [PATCH 06/34] Rewrote code for module.json creation

---
 nf_core/modules/module_utils.py     | 73 +++++++++++++++++++++++++++++
 nf_core/modules/pipeline_modules.py | 34 +-------------
 2 files changed, 75 insertions(+), 32 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 07aeb1e00..ed77aacf8 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -1,8 +1,13 @@
+import glob
+import json
+import os
 import requests
 import sys
 import logging
 import nf_core.utils
 
+from .pipeline_modules import ModulesRepo
+
 log = logging.getLogger(__name__)
 
 
@@ -20,3 +25,71 @@ def get_module_git_log(module_name):
         sys.exit(1)
     else:
         raise SystemError(f"Unable to fetch commit SHA for module {module_name}")
+
+
+def create_modules_json(pipeline_dir):
+    pipeline_config = nf_core.utils.fetch_wf_config(pipeline_dir)
+    pipeline_name = pipeline_config["manifest.name"]
+    pipeline_url = pipeline_config["manifest.homePage"]
+    modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "modules": {}}
+    module_paths = glob.glob(f"{pipeline_dir}/modules/nf-core/software/*")
+    module_names = [path.replace(f"{pipeline_dir}/modules/nf-core/software/", "") for path in module_paths]
+    module_repo = ModulesRepo()
+    for module_name, module_path in zip(module_names, module_paths):
+        try:
+            commit_shas = [commit["git_sha"] for commit in get_module_git_log(module_name)]
+            correct_commit_sha = find_correct_commit_sha(module_name, module_path, module_repo, commit_shas)
+            modules_json["modules"][module_name] = {"git_sha": correct_commit_sha}
+        except SystemError as e:
+            log.error(e)
+            log.error("Will not create 'modules.json' file")
+            sys.exit(1)
+    modules_json_path = os.path.join(pipeline_dir, "modules.json")
+    with open(modules_json_path, "w") as fh:
+        json.dump(modules_json, fh, indent=4)
+
+
+def find_correct_commit_sha(module_name, module_path, modules_repo, commit_shas):
+    files_to_check = ["main.nf", "functions.nf", "meta.yml"]
+    local_file_contents = [None, None, None]
+    for i, file in enumerate(files_to_check):
+        try:
+            local_file_contents[i] = open(os.path.join(module_path, file), "r").read()
+        except FileNotFoundError as e:
+            log.debug(f"Could not open file: {os.path.join(module_path, file)}")
+            continue
+    for commit_sha in commit_shas:
+        if local_module_equal_to_commit(local_file_contents, module_name, modules_repo, commit_sha):
+            return commit_sha
+    return None
+
+
+def local_module_equal_to_commit(local_files, module_name, module_path, modules_repo, commit_sha):
+    files_to_check = ["main.nf", "functions.nf", "meta.yml"]
+    files_are_equal = [False, False, False]
+    remote_copies = [None, None, None]
+
+    module_base_url = (
+        f"https://raw.githubusercontent.com/{modules_repo.name}/{modules_repo.branch}/software/{module_name}"
+    )
+    for i, file in enumerate(files_to_check):
+        # Download remote copy and compare
+        api_url = f"{module_base_url}/{file}/ref={commit_sha}"
+        r = requests.get(url=api_url)
+
+        if r.status_code != 200:
+            log.error(f"Could not download remote copy of file module {module_name}/{file}")
+        else:
+            try:
+                remote_copy = r.content.decode("utf-8")
+                remote_copies[i] = remote_copy
+
+            except UnicodeDecodeError as e:
+                log.error(f"Could not decode remote copy of {file} for the {module_name} module")
+        # Compare the contents of the files.
+        # If the file is missing from both the local and remote repo
+        # we will get the comparision None == None
+        if local_files[i] == remote_copy:
+            files_are_equal[i] = True
+
+    return all(files_are_equal)
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 5b6af8eb6..8d25b374b 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -32,6 +32,7 @@
 import sys
 
 import nf_core.utils
+from .module_utils import create_modules_json
 
 log = logging.getLogger(__name__)
 
@@ -355,35 +356,4 @@ def has_modules_file(self):
         modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
         if not os.path.exists(modules_json_path):
             log.info("Creating missing 'module.json' file.")
-            pipeline_config = nf_core.utils.fetch_wf_config(self.pipeline_dir)
-            pipeline_name = pipeline_config["manifest.name"]
-            pipeline_url = pipeline_config["manifest.homePage"]
-            modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "modules": {}}
-            module_names = [
-                path.replace(f"{self.pipeline_dir}/modules/nf-core/software/", "")
-                for path in glob.glob(f"{self.pipeline_dir}/modules/nf-core/software/*")
-            ]
-            for module_name in module_names:
-                try:
-                    commit_sha = self.get_module_commit_sha(module_name)
-                    modules_json["modules"][module_name] = {"git_sha": commit_sha}
-                except SystemError as e:
-                    log.error(e)
-                    log.error("Will not create 'modules.json' file")
-                    sys.exit(1)
-            modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
-            with open(modules_json_path, "w") as fh:
-                json.dump(modules_json, fh, indent=4)
-
-    def get_module_commit_sha(self, module_name):
-        """Fetches the latests commit SHA for the requested module"""
-        api_url = f'https://api.github.com/repos/nf-core/modules/commits/master?q={{path="software/{module_name}"}}'
-        response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
-        if response.status_code == 200:
-            json_response = response.json()
-            return json_response["sha"]
-        elif response.status_code == 404:
-            log.error(f"Module '{module_name}' not found in 'nf-core/modules/'\n{api_url}")
-            sys.exit(1)
-        else:
-            raise SystemError(f"Unable to fetch commit SHA for module {module_name}")
+            create_modules_json(self.pipeline_dir)

From 2c9a39edea887d58e9cdbafa80c0037bd7e00f85 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Fri, 18 Jun 2021 17:32:52 +0200
Subject: [PATCH 07/34] Moved ModulesRepo to separate file

---
 nf_core/modules/module_utils.py     |   2 +-
 nf_core/modules/modules_repo.py     | 111 ++++++++++++++++++++++++++++
 nf_core/modules/pipeline_modules.py | 104 +-------------------------
 3 files changed, 113 insertions(+), 104 deletions(-)
 create mode 100644 nf_core/modules/modules_repo.py

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index ed77aacf8..03599f30e 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -6,7 +6,7 @@
 import logging
 import nf_core.utils
 
-from .pipeline_modules import ModulesRepo
+from .modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)
 
diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py
new file mode 100644
index 000000000..7a6d35a5f
--- /dev/null
+++ b/nf_core/modules/modules_repo.py
@@ -0,0 +1,111 @@
+import os
+import requests
+import base64
+import sys
+import logging
+import nf_core.utils
+
+log = logging.getLogger(__name__)
+
+
+class ModulesRepo(object):
+    """
+    An object to store details about the repository being used for modules.
+
+    Used by the `nf-core modules` top-level command with -r and -b flags,
+    so that this can be used in the same way by all sucommands.
+    """
+
+    def __init__(self, repo="nf-core/modules", branch="master"):
+        self.name = repo
+        self.branch = branch
+        self.modules_file_tree = {}
+        self.modules_current_hash = None
+        self.modules_avail_module_names = []
+
+    def get_modules_file_tree(self):
+        """
+        Fetch the file list from the repo, using the GitHub API
+
+        Sets self.modules_file_tree
+             self.modules_current_hash
+             self.modules_avail_module_names
+        """
+        api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch)
+        r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
+        if r.status_code == 404:
+            log.error("Repository / branch not found: {} ({})\n{}".format(self.name, self.branch, api_url))
+            sys.exit(1)
+        elif r.status_code != 200:
+            raise SystemError(
+                "Could not fetch {} ({}) tree: {}\n{}".format(self.name, self.branch, r.status_code, api_url)
+            )
+
+        result = r.json()
+        assert result["truncated"] == False
+
+        self.modules_current_hash = result["sha"]
+        self.modules_file_tree = result["tree"]
+        for f in result["tree"]:
+            if f["path"].startswith("software/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]:
+                # remove software/ and /main.nf
+                self.modules_avail_module_names.append(f["path"][9:-8])
+
+    def get_module_file_urls(self, module):
+        """Fetch list of URLs for a specific module
+
+        Takes the name of a module and iterates over the GitHub repo file tree.
+        Loops over items that are prefixed with the path 'software/<module_name>' and ignores
+        anything that's not a blob. Also ignores the test/ subfolder.
+
+        Returns a dictionary with keys as filenames and values as GitHub API URIs.
+        These can be used to then download file contents.
+
+        Args:
+            module (string): Name of module for which to fetch a set of URLs
+
+        Returns:
+            dict: Set of files and associated URLs as follows:
+
+            {
+                'software/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3',
+                'software/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651'
+            }
+        """
+        results = {}
+        for f in self.modules_file_tree:
+            if not f["path"].startswith("software/{}".format(module)):
+                continue
+            if f["type"] != "blob":
+                continue
+            if "/test/" in f["path"]:
+                continue
+            results[f["path"]] = f["url"]
+        return results
+
+    def download_gh_file(self, dl_filename, api_url):
+        """Download a file from GitHub using the GitHub API
+
+        Args:
+            dl_filename (string): Path to save file to
+            api_url (string): GitHub API URL for file
+
+        Raises:
+            If a problem, raises an error
+        """
+
+        # Make target directory if it doesn't already exist
+        dl_directory = os.path.dirname(dl_filename)
+        if not os.path.exists(dl_directory):
+            os.makedirs(dl_directory)
+
+        # Call the GitHub API
+        r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
+        if r.status_code != 200:
+            raise SystemError("Could not fetch {} file: {}\n {}".format(self.name, r.status_code, api_url))
+        result = r.json()
+        file_contents = base64.b64decode(result["content"])
+
+        # Write the file contents
+        with open(dl_filename, "wb") as fh:
+            fh.write(file_contents)
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 8d25b374b..337dc1c57 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -33,113 +33,11 @@
 
 import nf_core.utils
 from .module_utils import create_modules_json
+from .modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)
 
 
-class ModulesRepo(object):
-    """
-    An object to store details about the repository being used for modules.
-
-    Used by the `nf-core modules` top-level command with -r and -b flags,
-    so that this can be used in the same way by all sucommands.
-    """
-
-    def __init__(self, repo="nf-core/modules", branch="master"):
-        self.name = repo
-        self.branch = branch
-        self.modules_file_tree = {}
-        self.modules_current_hash = None
-        self.modules_avail_module_names = []
-
-    def get_modules_file_tree(self):
-        """
-        Fetch the file list from the repo, using the GitHub API
-
-        Sets self.modules_file_tree
-             self.modules_current_hash
-             self.modules_avail_module_names
-        """
-        api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch)
-        r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
-        if r.status_code == 404:
-            log.error("Repository / branch not found: {} ({})\n{}".format(self.name, self.branch, api_url))
-            sys.exit(1)
-        elif r.status_code != 200:
-            raise SystemError(
-                "Could not fetch {} ({}) tree: {}\n{}".format(self.name, self.branch, r.status_code, api_url)
-            )
-
-        result = r.json()
-        assert result["truncated"] == False
-
-        self.modules_current_hash = result["sha"]
-        self.modules_file_tree = result["tree"]
-        for f in result["tree"]:
-            if f["path"].startswith("software/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]:
-                # remove software/ and /main.nf
-                self.modules_avail_module_names.append(f["path"][9:-8])
-
-    def get_module_file_urls(self, module):
-        """Fetch list of URLs for a specific module
-
-        Takes the name of a module and iterates over the GitHub repo file tree.
-        Loops over items that are prefixed with the path 'software/<module_name>' and ignores
-        anything that's not a blob. Also ignores the test/ subfolder.
-
-        Returns a dictionary with keys as filenames and values as GitHub API URIs.
-        These can be used to then download file contents.
-
-        Args:
-            module (string): Name of module for which to fetch a set of URLs
-
-        Returns:
-            dict: Set of files and associated URLs as follows:
-
-            {
-                'software/fastqc/main.nf': 'https://api.github.com/repos/nf-core/modules/git/blobs/65ba598119206a2b851b86a9b5880b5476e263c3',
-                'software/fastqc/meta.yml': 'https://api.github.com/repos/nf-core/modules/git/blobs/0d5afc23ba44d44a805c35902febc0a382b17651'
-            }
-        """
-        results = {}
-        for f in self.modules_file_tree:
-            if not f["path"].startswith("software/{}".format(module)):
-                continue
-            if f["type"] != "blob":
-                continue
-            if "/test/" in f["path"]:
-                continue
-            results[f["path"]] = f["url"]
-        return results
-
-    def download_gh_file(self, dl_filename, api_url):
-        """Download a file from GitHub using the GitHub API
-
-        Args:
-            dl_filename (string): Path to save file to
-            api_url (string): GitHub API URL for file
-
-        Raises:
-            If a problem, raises an error
-        """
-
-        # Make target directory if it doesn't already exist
-        dl_directory = os.path.dirname(dl_filename)
-        if not os.path.exists(dl_directory):
-            os.makedirs(dl_directory)
-
-        # Call the GitHub API
-        r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
-        if r.status_code != 200:
-            raise SystemError("Could not fetch {} file: {}\n {}".format(self.name, r.status_code, api_url))
-        result = r.json()
-        file_contents = base64.b64decode(result["content"])
-
-        # Write the file contents
-        with open(dl_filename, "wb") as fh:
-            fh.write(file_contents)
-
-
 class PipelineModules(object):
     def __init__(self):
         """

From 278fe863ebea46511f7a7b27bf8ee1bb92ae1294 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com>
Date: Fri, 18 Jun 2021 17:36:17 +0200
Subject: [PATCH 08/34] Apply suggestions from code review

Co-authored-by: Kevin Menden <kevin.menden@live.com>
---
 nf_core/modules/pipeline_modules.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 337dc1c57..8ae8dec6e 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -147,8 +147,7 @@ def install(self, module=None):
         try:
             commit_sha = self.get_module_commit_sha(module)
         except SystemError as e:
-            log.error(e)
-            log.error(f"Will remove module '{module}'")
+            log.error(f"Could not fetch `git_sha` for module '{module}': {e}")
             # Remove the module
             try:
                 shutil.rmtree(module_dir)

From a353a4399dd0909bb574bef802fe668d9d19d3b3 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Sat, 19 Jun 2021 22:02:38 +0200
Subject: [PATCH 09/34] Bug fixes after testing with rnaseq pipeline

---
 nf_core/modules/module_utils.py | 45 ++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 03599f30e..8ff071087 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -4,6 +4,7 @@
 import requests
 import sys
 import logging
+
 import nf_core.utils
 
 from .modules_repo import ModulesRepo
@@ -12,14 +13,15 @@
 
 
 def get_module_git_log(module_name):
-    """Fetches the git log_for the requested module"""
-    api_url = f'https://api.github.com/repos/nf-core/modules/commits?q={{sha=master, path="software/{module_name}"}}'
+    """Fetches the commit history the requested module"""
+    api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}"
     response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
     if response.status_code == 200:
-        commits = [
-            {"git_sha": commit["sha"], "trunc_message": commit["commit"]["message"]} for commit in response.json()
+        # Return the commit SHAs and the first line of the commit message
+        return [
+            {"git_sha": commit["sha"], "trunc_message": commit["commit"]["message"].partition("\n")[0]}
+            for commit in response.json()
         ]
-        return commits
     elif response.status_code == 404:
         log.error(f"Module '{module_name}' not found in 'nf-core/modules/'\n{api_url}")
         sys.exit(1)
@@ -28,11 +30,15 @@ def get_module_git_log(module_name):
 
 
 def create_modules_json(pipeline_dir):
+    """Create the modules.json files"""
     pipeline_config = nf_core.utils.fetch_wf_config(pipeline_dir)
     pipeline_name = pipeline_config["manifest.name"]
     pipeline_url = pipeline_config["manifest.homePage"]
     modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "modules": {}}
-    module_paths = glob.glob(f"{pipeline_dir}/modules/nf-core/software/*")
+    all_module_file_paths = glob.glob(f"{pipeline_dir}/modules/nf-core/software/**/*", recursive=True)
+
+    # Extract the module paths from the file paths
+    module_paths = list(set(map(os.path.dirname, filter(os.path.isfile, all_module_file_paths))))
     module_names = [path.replace(f"{pipeline_dir}/modules/nf-core/software/", "") for path in module_paths]
     module_repo = ModulesRepo()
     for module_name, module_path in zip(module_names, module_paths):
@@ -49,7 +55,12 @@ def create_modules_json(pipeline_dir):
         json.dump(modules_json, fh, indent=4)
 
 
+def get_module_paths(pipeline_dir):
+    base_dir = f"{pipeline_dir}/modules/nf-core/software"
+
+
 def find_correct_commit_sha(module_name, module_path, modules_repo, commit_shas):
+    """Returns the SHA for the latest commit where the local files equal the remote files"""
     files_to_check = ["main.nf", "functions.nf", "meta.yml"]
     local_file_contents = [None, None, None]
     for i, file in enumerate(files_to_check):
@@ -64,32 +75,30 @@ def find_correct_commit_sha(module_name, module_path, modules_repo, commit_shas)
     return None
 
 
-def local_module_equal_to_commit(local_files, module_name, module_path, modules_repo, commit_sha):
+def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_sha):
+    """Compares the local module files to the module files for the given commit sha"""
     files_to_check = ["main.nf", "functions.nf", "meta.yml"]
     files_are_equal = [False, False, False]
     remote_copies = [None, None, None]
 
-    module_base_url = (
-        f"https://raw.githubusercontent.com/{modules_repo.name}/{modules_repo.branch}/software/{module_name}"
-    )
+    module_base_url = f"https://raw.githubusercontent.com/{modules_repo.name}/{commit_sha}/software/{module_name}"
     for i, file in enumerate(files_to_check):
         # Download remote copy and compare
-        api_url = f"{module_base_url}/{file}/ref={commit_sha}"
+        api_url = f"{module_base_url}/{file}"
         r = requests.get(url=api_url)
-
         if r.status_code != 200:
-            log.error(f"Could not download remote copy of file module {module_name}/{file}")
+            log.debug(f"Could not download remote copy of file module {module_name}/{file}")
+            log.debug(api_url)
         else:
             try:
-                remote_copy = r.content.decode("utf-8")
-                remote_copies[i] = remote_copy
-
+                remote_copies[i] = r.content.decode("utf-8")
             except UnicodeDecodeError as e:
-                log.error(f"Could not decode remote copy of {file} for the {module_name} module")
+                log.debug(f"Could not decode remote copy of {file} for the {module_name} module")
+
         # Compare the contents of the files.
         # If the file is missing from both the local and remote repo
         # we will get the comparision None == None
-        if local_files[i] == remote_copy:
+        if local_files[i] == remote_copies[i]:
             files_are_equal[i] = True
 
     return all(files_are_equal)

From a9a8d17667d0b263c775ce59f7fb566f5582a235 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Sun, 20 Jun 2021 11:06:24 +0200
Subject: [PATCH 10/34] Added support for pagination of commits

---
 nf_core/modules/module_utils.py | 38 +++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 8ff071087..eb887d107 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -4,6 +4,9 @@
 import requests
 import sys
 import logging
+from itertools import count
+
+from requests import api
 
 import nf_core.utils
 
@@ -12,16 +15,22 @@
 log = logging.getLogger(__name__)
 
 
-def get_module_git_log(module_name):
+def get_module_git_log(module_name, per_page=30, page_nbr=1):
     """Fetches the commit history the requested module"""
-    api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}"
+    api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}&per_page={per_page}&page={page_nbr}"
     response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
     if response.status_code == 200:
-        # Return the commit SHAs and the first line of the commit message
-        return [
-            {"git_sha": commit["sha"], "trunc_message": commit["commit"]["message"].partition("\n")[0]}
-            for commit in response.json()
-        ]
+        commits = response.json()
+
+        if len(commits) == 0:
+            log.debug(f"Reached end of commit history for '{module_name}'")
+            raise SystemError(f"Unable to fetch commit SHA for module {module_name}")
+        else:
+            # Return the commit SHAs and the first line of the commit message
+            return [
+                {"git_sha": commit["sha"], "trunc_message": commit["commit"]["message"].partition("\n")[0]}
+                for commit in commits
+            ]
     elif response.status_code == 404:
         log.error(f"Module '{module_name}' not found in 'nf-core/modules/'\n{api_url}")
         sys.exit(1)
@@ -43,8 +52,19 @@ def create_modules_json(pipeline_dir):
     module_repo = ModulesRepo()
     for module_name, module_path in zip(module_names, module_paths):
         try:
-            commit_shas = [commit["git_sha"] for commit in get_module_git_log(module_name)]
-            correct_commit_sha = find_correct_commit_sha(module_name, module_path, module_repo, commit_shas)
+            # Find the correct commit SHA for the local files.
+            # We iterate over the commit log pages until we either
+            # find a matching commit or we reach the end of the commits
+            correct_commit_sha = None
+            commit_page_nbr = 1
+            while correct_commit_sha is None:
+
+                commit_shas = [
+                    commit["git_sha"] for commit in get_module_git_log(module_name, page_nbr=commit_page_nbr)
+                ]
+                correct_commit_sha = find_correct_commit_sha(module_name, module_path, module_repo, commit_shas)
+                commit_page_nbr += 1
+
             modules_json["modules"][module_name] = {"git_sha": correct_commit_sha}
         except SystemError as e:
             log.error(e)

From e04cf4e92abdedcb931f6715f5cce72580912443 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Sun, 20 Jun 2021 13:02:58 +0200
Subject: [PATCH 11/34] Update function descriptions

---
 nf_core/modules/module_utils.py | 44 +++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index eb887d107..282a3a3fe 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -16,7 +16,15 @@
 
 
 def get_module_git_log(module_name, per_page=30, page_nbr=1):
-    """Fetches the commit history the requested module"""
+    """
+    Fetches the commit history the requested module
+    Args:
+        module_name (str): Name of module
+        per_page (int): Number of commits per page returned by API
+        page_nbr (int): Page number of the retrieved commits
+    Returns:
+        [ dict ]: List of commit SHAs and associated (truncated) message
+    """
     api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}&per_page={per_page}&page={page_nbr}"
     response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
     if response.status_code == 200:
@@ -39,7 +47,12 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1):
 
 
 def create_modules_json(pipeline_dir):
-    """Create the modules.json files"""
+    """
+    Create the modules.json files
+
+    Args:
+        pipeline_dir (str): The directory where the `modules.json` should be created
+    """
     pipeline_config = nf_core.utils.fetch_wf_config(pipeline_dir)
     pipeline_name = pipeline_config["manifest.name"]
     pipeline_url = pipeline_config["manifest.homePage"]
@@ -75,12 +88,18 @@ def create_modules_json(pipeline_dir):
         json.dump(modules_json, fh, indent=4)
 
 
-def get_module_paths(pipeline_dir):
-    base_dir = f"{pipeline_dir}/modules/nf-core/software"
-
-
 def find_correct_commit_sha(module_name, module_path, modules_repo, commit_shas):
-    """Returns the SHA for the latest commit where the local files equal the remote files"""
+    """
+    Returns the SHA for the latest commit where the local files are identical to the remote files
+    Args:
+        module_name (str): Name of module
+        module_path (str): Path to module in local repo
+        module_repo (str): Remote repo for module
+        commit_shas ([ str ]): List of commit SHAs for module, sorted in descending order
+    Returns:
+        commit_sha (str): The latest commit SHA where local files are identical to remote files
+    """
+
     files_to_check = ["main.nf", "functions.nf", "meta.yml"]
     local_file_contents = [None, None, None]
     for i, file in enumerate(files_to_check):
@@ -96,7 +115,16 @@ def find_correct_commit_sha(module_name, module_path, modules_repo, commit_shas)
 
 
 def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_sha):
-    """Compares the local module files to the module files for the given commit sha"""
+    """
+    Compares the local module files to the module files for the given commit sha
+    Args:
+        local_files ([ str ]): Contents of local files. `None` if files doesn't exist
+        module_name (str): Name of module
+        module_repo (str): Remote repo for module
+        commit_sha (str): Commit SHA for remote version to compare against local version
+    Returns:
+        bool: Whether all local files are identical to remote version
+    """
     files_to_check = ["main.nf", "functions.nf", "meta.yml"]
     files_are_equal = [False, False, False]
     remote_copies = [None, None, None]

From 0dbe865510bb68205e0fa3c1ef79fbe482bcf65a Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Mon, 21 Jun 2021 09:16:46 +0200
Subject: [PATCH 12/34] Implemented installation of different pipeline versions

---
 nf_core/modules/module_utils.py     | 25 +++++++++++++++++++++++--
 nf_core/modules/modules_repo.py     |  7 +++++--
 nf_core/modules/pipeline_modules.py | 29 +++++++----------------------
 3 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 282a3a3fe..932274fb1 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -4,6 +4,7 @@
 import requests
 import sys
 import logging
+import questionary
 from itertools import count
 
 from requests import api
@@ -31,8 +32,7 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1):
         commits = response.json()
 
         if len(commits) == 0:
-            log.debug(f"Reached end of commit history for '{module_name}'")
-            raise SystemError(f"Unable to fetch commit SHA for module {module_name}")
+            raise SystemError(f"Reached end of commit history for '{module_name}'")
         else:
             # Return the commit SHAs and the first line of the commit message
             return [
@@ -150,3 +150,24 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_
             files_are_equal[i] = True
 
     return all(files_are_equal)
+
+
+def prompt_module_version_sha(module):
+    older_commits_choice = questionary.Choice(
+        title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value=""
+    )
+
+    git_sha = ""
+    page_nbr = 1
+    while git_sha is "":
+        commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr)
+        commit_titles = []
+        for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
+            tag_display = [("fg:ansiblue", f"{title}  "), ("class:choice-default", "")]
+            commit_titles.append(questionary.Choice(title=tag_display, value=sha))
+        choices = commit_titles + [older_commits_choice]
+        git_sha = questionary.select(
+            f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style
+        ).ask()
+        page_nbr += 1
+    return git_sha
diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py
index 7a6d35a5f..f0b90266e 100644
--- a/nf_core/modules/modules_repo.py
+++ b/nf_core/modules/modules_repo.py
@@ -51,14 +51,14 @@ def get_modules_file_tree(self):
                 # remove software/ and /main.nf
                 self.modules_avail_module_names.append(f["path"][9:-8])
 
-    def get_module_file_urls(self, module):
+    def get_module_file_urls(self, module, commit=""):
         """Fetch list of URLs for a specific module
 
         Takes the name of a module and iterates over the GitHub repo file tree.
         Loops over items that are prefixed with the path 'software/<module_name>' and ignores
         anything that's not a blob. Also ignores the test/ subfolder.
 
-        Returns a dictionary with keys as filenames and values as GitHub API URIs.
+        Returns a dictionary with keys as filenames and values as GitHub API URLs.
         These can be used to then download file contents.
 
         Args:
@@ -81,6 +81,9 @@ def get_module_file_urls(self, module):
             if "/test/" in f["path"]:
                 continue
             results[f["path"]] = f["url"]
+        if commit != "":
+            for path in results:
+                results[path] = f"https://api.github.com/repos/nf-core/modules/contents/{path}?ref={commit}"
         return results
 
     def download_gh_file(self, dl_filename, api_url):
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 8ae8dec6e..4d7109c74 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -32,7 +32,7 @@
 import sys
 
 import nf_core.utils
-from .module_utils import create_modules_json
+from .module_utils import create_modules_json, get_module_git_log, prompt_module_version_sha
 from .modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)
@@ -108,6 +108,11 @@ def install(self, module=None):
                 choices=self.modules_repo.modules_avail_module_names,
                 style=nf_core.utils.nfcore_question_style,
             ).ask()
+        try:
+            commit_sha = prompt_module_version_sha(module)
+        except SystemError as e:
+            log.error(e)
+            sys.exit(1)
 
         log.info("Installing {}".format(module))
 
@@ -132,7 +137,7 @@ def install(self, module=None):
             return False
 
         # Download module files
-        files = self.modules_repo.get_module_file_urls(module)
+        files = self.modules_repo.get_module_file_urls(module, commit_sha)
         log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys())))
         for filename, api_url in files.items():
             split_filename = filename.split("/")
@@ -144,26 +149,6 @@ def install(self, module=None):
         modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
         with open(modules_json_path, "r") as fh:
             modules_json = json.load(fh)
-        try:
-            commit_sha = self.get_module_commit_sha(module)
-        except SystemError as e:
-            log.error(f"Could not fetch `git_sha` for module '{module}': {e}")
-            # Remove the module
-            try:
-                shutil.rmtree(module_dir)
-                # Try cleaning up empty parent if tool/subtool and tool/ is empty
-                if module.count("/") > 0:
-                    parent_dir = os.path.dirname(module_dir)
-                    try:
-                        os.rmdir(parent_dir)
-                    except OSError:
-                        log.debug(f"Parent directory not empty: '{parent_dir}'")
-                    else:
-                        log.debug(f"Deleted orphan tool directory: '{parent_dir}'")
-                return False
-            except OSError as e:
-                log.error("Could not remove module: {}".format(e))
-                return False
 
         modules_json["modules"][module] = {"git_sha": commit_sha}
         with open(modules_json_path, "w") as fh:

From ef738685e736741f9b61da3df399d21f76bf120a Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Mon, 21 Jun 2021 16:21:38 +0200
Subject: [PATCH 13/34] Added new flag '--latest' to install the latest version
 of module without interactive prompt

---
 nf_core/__main__.py                 |  5 +++--
 nf_core/modules/module_utils.py     |  2 +-
 nf_core/modules/pipeline_modules.py | 23 +++++++++++++----------
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 485a6caba..a7a8b91a1 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -380,7 +380,8 @@ def list(ctx, pipeline_dir, json):
 @click.pass_context
 @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.option("-t", "--tool", type=str, metavar="<tool> or <tool/subtool>")
-def install(ctx, pipeline_dir, tool):
+@click.option("-l", "--latest", is_flag=True, default=False, help="Install the latest version of the module")
+def install(ctx, pipeline_dir, tool, latest):
     """
     Add a DSL2 software wrapper module to a pipeline.
 
@@ -391,7 +392,7 @@ def install(ctx, pipeline_dir, tool):
         mods = nf_core.modules.PipelineModules()
         mods.modules_repo = ctx.obj["modules_repo_obj"]
         mods.pipeline_dir = pipeline_dir
-        mods.install(tool)
+        mods.install(tool, latest)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 932274fb1..3d7ab3991 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -168,6 +168,6 @@ def prompt_module_version_sha(module):
         choices = commit_titles + [older_commits_choice]
         git_sha = questionary.select(
             f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style
-        ).ask()
+        ).unsafe_ask()
         page_nbr += 1
     return git_sha
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 4d7109c74..1dac40b00 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -94,7 +94,7 @@ def list_modules(self, print_json=False):
             return json.dumps(modules, sort_keys=True, indent=4)
         return table
 
-    def install(self, module=None):
+    def install(self, module=None, latest=False):
 
         # Check whether pipelines is valid
         self.has_valid_pipeline()
@@ -107,13 +107,16 @@ def install(self, module=None):
                 "Tool name:",
                 choices=self.modules_repo.modules_avail_module_names,
                 style=nf_core.utils.nfcore_question_style,
-            ).ask()
-        try:
-            commit_sha = prompt_module_version_sha(module)
-        except SystemError as e:
-            log.error(e)
-            sys.exit(1)
-
+            ).unsafe_ask()
+        if latest:
+            # Fetch the latest commit to the module
+            version = get_module_git_log(module, per_page=1, page_nbr=1)[0]["git_sha"]
+        else:
+            try:
+                version = prompt_module_version_sha(module)
+            except SystemError as e:
+                log.error(e)
+                sys.exit(1)
         log.info("Installing {}".format(module))
 
         # Check that the supplied name is an available module
@@ -137,7 +140,7 @@ def install(self, module=None):
             return False
 
         # Download module files
-        files = self.modules_repo.get_module_file_urls(module, commit_sha)
+        files = self.modules_repo.get_module_file_urls(module, version)
         log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys())))
         for filename, api_url in files.items():
             split_filename = filename.split("/")
@@ -150,7 +153,7 @@ def install(self, module=None):
         with open(modules_json_path, "r") as fh:
             modules_json = json.load(fh)
 
-        modules_json["modules"][module] = {"git_sha": commit_sha}
+        modules_json["modules"][module] = {"git_sha": version}
         with open(modules_json_path, "w") as fh:
             json.dump(modules_json, fh, indent=4)
 

From 1ee746d7e4399612498b3dc81552448d2009e38e Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Mon, 21 Jun 2021 16:26:50 +0200
Subject: [PATCH 14/34] Update module test to handle versioning

---
 tests/test_modules.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/test_modules.py b/tests/test_modules.py
index 88a5bc452..f65e213fd 100644
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -65,46 +65,46 @@ def test_modules_list(self):
     def test_modules_install_nopipeline(self):
         """Test installing a module - no pipeline given"""
         self.mods.pipeline_dir = None
-        assert self.mods.install("foo") is False
+        assert self.mods.install("foo", latest=True) is False
 
     def test_modules_install_emptypipeline(self):
         """Test installing a module - empty dir given"""
         self.mods.pipeline_dir = tempfile.mkdtemp()
         with pytest.raises(UserWarning) as excinfo:
-            self.mods.install("foo")
+            self.mods.install("foo", latest=True)
         assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value)
 
     def test_modules_install_nomodule(self):
         """Test installing a module - unrecognised module given"""
-        assert self.mods.install("foo") is False
+        assert self.mods.install("foo", latest=True) is False
 
     def test_modules_install_trimgalore(self):
         """Test installing a module - TrimGalore!"""
-        assert self.mods.install("trimgalore") is not False
+        assert self.mods.install("trimgalore", latest=True) is not False
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "trimgalore")
         assert os.path.exists(module_path)
 
     def test_modules_install_trimgalore_alternative_source(self):
         """Test installing a module from a different source repository - TrimGalore!"""
-        assert self.mods_alt.install("trimgalore") is not False
+        assert self.mods_alt.install("trimgalore", latest=True) is not False
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "external", "trimgalore")
         assert os.path.exists(module_path)
 
     def test_modules_install_trimgalore_twice(self):
         """Test installing a module - TrimGalore! already there"""
-        self.mods.install("trimgalore")
-        assert self.mods.install("trimgalore") is False
+        self.mods.install("trimgalore", latest=True)
+        assert self.mods.install("trimgalore", latest=True) is False
 
     def test_modules_remove_trimgalore(self):
         """Test removing TrimGalore! module after installing it"""
-        self.mods.install("trimgalore")
+        self.mods.install("trimgalore", latest=True)
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "trimgalore")
         assert self.mods.remove("trimgalore")
         assert os.path.exists(module_path) is False
 
     def test_modules_remove_trimgalore_alternative_source(self):
         """Test removing TrimGalore! module after installing it from an alternative source"""
-        self.mods_alt.install("trimgalore")
+        self.mods_alt.install("trimgalore", latest=True)
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "external", "trimgalore")
         assert self.mods_alt.remove("trimgalore")
         assert os.path.exists(module_path) is False
@@ -115,7 +115,7 @@ def test_modules_remove_trimgalore_uninstalled(self):
 
     def test_modules_lint_trimgalore(self):
         """Test linting the TrimGalore! module"""
-        self.mods.install("trimgalore")
+        self.mods.install("trimgalore", latest=True)
         module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir)
         module_lint.lint(print_results=False, module="trimgalore")
         assert len(module_lint.passed) == 20

From 116f65be900b2e4b7b919ed175b790cf5ffe8592 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com>
Date: Mon, 21 Jun 2021 16:27:16 +0200
Subject: [PATCH 15/34] Update nf_core/modules/module_utils.py

Co-authored-by: Kevin Menden <kevin.menden@live.com>
---
 nf_core/modules/module_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 3d7ab3991..32764ad34 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -18,7 +18,7 @@
 
 def get_module_git_log(module_name, per_page=30, page_nbr=1):
     """
-    Fetches the commit history the requested module
+    Fetches the commit history the of requested module
     Args:
         module_name (str): Name of module
         per_page (int): Number of commits per page returned by API

From bfa93e4dab258391d1988e2c56b863cd304a8154 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Mon, 21 Jun 2021 16:34:34 +0200
Subject: [PATCH 16/34] Update test with new flag for install command

---
 .github/workflows/create-lint-wf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml
index a87a55819..0233e9db3 100644
--- a/.github/workflows/create-lint-wf.yml
+++ b/.github/workflows/create-lint-wf.yml
@@ -53,7 +53,7 @@ jobs:
         run: nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored --release
 
       - name: nf-core modules install
-        run: nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc
+        run: nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc --latest
 
       - name: Upload log file artifact
         if: ${{ always() }}

From 38b314d3e16b254249c363071819725737e01ca9 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Mon, 21 Jun 2021 17:03:59 +0200
Subject: [PATCH 17/34] Only display 'older commits' if there are any

---
 nf_core/modules/module_utils.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 32764ad34..eb0a6e976 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -32,7 +32,8 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1):
         commits = response.json()
 
         if len(commits) == 0:
-            raise SystemError(f"Reached end of commit history for '{module_name}'")
+            log.debug(f"Reached end of commit history for '{module_name}'")
+            return None
         else:
             # Return the commit SHAs and the first line of the commit message
             return [
@@ -46,6 +47,10 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1):
         raise SystemError(f"Unable to fetch commit SHA for module {module_name}")
 
 
+def module_git_log_has_next_page(module_name, per_page=30, page_nbr=1):
+    pass
+
+
 def create_modules_json(pipeline_dir):
     """
     Create the modules.json files
@@ -156,16 +161,19 @@ def prompt_module_version_sha(module):
     older_commits_choice = questionary.Choice(
         title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value=""
     )
-
     git_sha = ""
     page_nbr = 1
+    next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr)
     while git_sha is "":
-        commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr)
+        commits = next_page_commits
+        next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1)
         commit_titles = []
         for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
             tag_display = [("fg:ansiblue", f"{title}  "), ("class:choice-default", "")]
             commit_titles.append(questionary.Choice(title=tag_display, value=sha))
-        choices = commit_titles + [older_commits_choice]
+        choices = commit_titles
+        if next_page_commits is not None:
+            choices += [older_commits_choice]
         git_sha = questionary.select(
             f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style
         ).unsafe_ask()

From cfad229ed1d9c4309f9a67a07eaea37a5fed689c Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Mon, 21 Jun 2021 17:08:31 +0200
Subject: [PATCH 18/34] Remove redundant code

---
 nf_core/modules/module_utils.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index eb0a6e976..5e23fbaaf 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -47,10 +47,6 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1):
         raise SystemError(f"Unable to fetch commit SHA for module {module_name}")
 
 
-def module_git_log_has_next_page(module_name, per_page=30, page_nbr=1):
-    pass
-
-
 def create_modules_json(pipeline_dir):
     """
     Create the modules.json files

From 1a5c215ccdfe93dd71074fe3a37d66be8534a964 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Mon, 21 Jun 2021 18:31:53 +0200
Subject: [PATCH 19/34] Restriced commit log to only be shown from given date

---
 nf_core/modules/module_utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 5e23fbaaf..9ff335e67 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -16,17 +16,20 @@
 log = logging.getLogger(__name__)
 
 
-def get_module_git_log(module_name, per_page=30, page_nbr=1):
+def get_module_git_log(module_name, per_page=30, page_nbr=1, since="2020-11-25T00:00:00Z"):
     """
     Fetches the commit history the of requested module
     Args:
         module_name (str): Name of module
         per_page (int): Number of commits per page returned by API
         page_nbr (int): Page number of the retrieved commits
+        since (str): Only show commits later than this timestamp.
+        Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ.
+
     Returns:
         [ dict ]: List of commit SHAs and associated (truncated) message
     """
-    api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}&per_page={per_page}&page={page_nbr}"
+    api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}&per_page={per_page}&page={page_nbr}&since={since}"
     response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
     if response.status_code == 200:
         commits = response.json()

From bc6366e897079b48d653eeba4a98464e1e51e2a4 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 10:02:24 +0200
Subject: [PATCH 20/34] Minor fixes

---
 nf_core/modules/module_utils.py     | 10 +++++-----
 nf_core/modules/pipeline_modules.py |  3 ++-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 9ff335e67..0042cbfa7 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -36,7 +36,7 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1, since="2020-11-25T0
 
         if len(commits) == 0:
             log.debug(f"Reached end of commit history for '{module_name}'")
-            return None
+            return []
         else:
             # Return the commit SHAs and the first line of the commit message
             return [
@@ -129,6 +129,7 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_
     Returns:
         bool: Whether all local files are identical to remote version
     """
+
     files_to_check = ["main.nf", "functions.nf", "meta.yml"]
     files_are_equal = [False, False, False]
     remote_copies = [None, None, None]
@@ -166,12 +167,11 @@ def prompt_module_version_sha(module):
     while git_sha is "":
         commits = next_page_commits
         next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1)
-        commit_titles = []
+        choices = []
         for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
             tag_display = [("fg:ansiblue", f"{title}  "), ("class:choice-default", "")]
-            commit_titles.append(questionary.Choice(title=tag_display, value=sha))
-        choices = commit_titles
-        if next_page_commits is not None:
+            choices.append(questionary.Choice(title=tag_display, value=sha))
+        if len(next_page_commits) > 0:
             choices += [older_commits_choice]
         git_sha = questionary.select(
             f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 1dac40b00..3a909658d 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -109,7 +109,7 @@ def install(self, module=None, latest=False):
                 style=nf_core.utils.nfcore_question_style,
             ).unsafe_ask()
         if latest:
-            # Fetch the latest commit to the module
+            # Fetch the latest commit for the module
             version = get_module_git_log(module, per_page=1, page_nbr=1)[0]["git_sha"]
         else:
             try:
@@ -156,6 +156,7 @@ def install(self, module=None, latest=False):
         modules_json["modules"][module] = {"git_sha": version}
         with open(modules_json_path, "w") as fh:
             json.dump(modules_json, fh, indent=4)
+        return True
 
     def update(self, module, force=False):
         log.error("This command is not yet implemented")

From ad7482dec0969ebc8bda4c6d089338e6c6a4a707 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 10:17:49 +0200
Subject: [PATCH 21/34] Fix index of out range error

---
 nf_core/modules/pipeline_modules.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 3a909658d..6862d5ee4 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -110,7 +110,11 @@ def install(self, module=None, latest=False):
             ).unsafe_ask()
         if latest:
             # Fetch the latest commit for the module
-            version = get_module_git_log(module, per_page=1, page_nbr=1)[0]["git_sha"]
+            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+            if len(git_log) == 0:
+                log.error(f"Was unable to fetch version of module '{module}'")
+                return False
+            version = git_log[0]["git_sha"]
         else:
             try:
                 version = prompt_module_version_sha(module)

From 2a07d527b75621a8ac58751f2df8d8c95b684c1e Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 11:25:28 +0200
Subject: [PATCH 22/34] Updated CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f8f0891ff..eefcb5e19 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,7 +17,7 @@
 
 * Update comment style of modules `functions.nf` template file [[#1076](https://github.com/nf-core/tools/issues/1076)]
 * Use Biocontainers API instead of quayi.io API for `nf-core modules create` [[#875](https://github.com/nf-core/tools/issues/875)]
-
+* Update `nf-core modules install` to handle different versions of modules  [#1116](https://github.com/nf-core/tools/pull/1116)
 #### Sync
 
 * Don't set the default value to `"null"` when a parameter is initialised as `null` in the config [[#1074](https://github.com/nf-core/tools/pull/1074)]

From ee5bbea6ae8ad99e5a2079168c97e32ceb7c02a4 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 11:33:01 +0200
Subject: [PATCH 23/34] Fix CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eefcb5e19..a72d63478 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@
 * Update comment style of modules `functions.nf` template file [[#1076](https://github.com/nf-core/tools/issues/1076)]
 * Use Biocontainers API instead of quayi.io API for `nf-core modules create` [[#875](https://github.com/nf-core/tools/issues/875)]
 * Update `nf-core modules install` to handle different versions of modules  [#1116](https://github.com/nf-core/tools/pull/1116)
+
 #### Sync
 
 * Don't set the default value to `"null"` when a parameter is initialised as `null` in the config [[#1074](https://github.com/nf-core/tools/pull/1074)]

From 1ea9be8574c1cda5ea957f8afb52a8818ef14617 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 13:41:18 +0200
Subject: [PATCH 24/34] Add '--force' option

---
 nf_core/__main__.py                 |   5 +-
 nf_core/modules/module_utils.py     |   1 +
 nf_core/modules/pipeline_modules.py | 117 ++++++++++++++++++++++------
 3 files changed, 97 insertions(+), 26 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index a7a8b91a1..6e10169f8 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -381,7 +381,8 @@ def list(ctx, pipeline_dir, json):
 @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.option("-t", "--tool", type=str, metavar="<tool> or <tool/subtool>")
 @click.option("-l", "--latest", is_flag=True, default=False, help="Install the latest version of the module")
-def install(ctx, pipeline_dir, tool, latest):
+@click.option("--force", is_flag=True, default=False, help="Force installation of module if module already exists")
+def install(ctx, pipeline_dir, tool, latest, force):
     """
     Add a DSL2 software wrapper module to a pipeline.
 
@@ -392,7 +393,7 @@ def install(ctx, pipeline_dir, tool, latest):
         mods = nf_core.modules.PipelineModules()
         mods.modules_repo = ctx.obj["modules_repo_obj"]
         mods.pipeline_dir = pipeline_dir
-        mods.install(tool, latest)
+        mods.install(tool, latest, force)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 0042cbfa7..0b22b0bb5 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -30,6 +30,7 @@ def get_module_git_log(module_name, per_page=30, page_nbr=1, since="2020-11-25T0
         [ dict ]: List of commit SHAs and associated (truncated) message
     """
     api_url = f"https://api.github.com/repos/nf-core/modules/commits?sha=master&path=software/{module_name}&per_page={per_page}&page={page_nbr}&since={since}"
+    log.debug(f"Fetching commit history of module '{module_name}' from github API")
     response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth())
     if response.status_code == 200:
         commits = response.json()
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 6862d5ee4..5e071c645 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -94,7 +94,7 @@ def list_modules(self, print_json=False):
             return json.dumps(modules, sort_keys=True, indent=4)
         return table
 
-    def install(self, module=None, latest=False):
+    def install(self, module=None, latest=False, force=False):
 
         # Check whether pipelines is valid
         self.has_valid_pipeline()
@@ -108,40 +108,112 @@ def install(self, module=None, latest=False):
                 choices=self.modules_repo.modules_avail_module_names,
                 style=nf_core.utils.nfcore_question_style,
             ).unsafe_ask()
-        if latest:
-            # Fetch the latest commit for the module
-            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
-            if len(git_log) == 0:
-                log.error(f"Was unable to fetch version of module '{module}'")
-                return False
-            version = git_log[0]["git_sha"]
-        else:
-            try:
-                version = prompt_module_version_sha(module)
-            except SystemError as e:
-                log.error(e)
-                sys.exit(1)
-        log.info("Installing {}".format(module))
 
         # Check that the supplied name is an available module
         if module not in self.modules_repo.modules_avail_module_names:
             log.error("Module '{}' not found in list of available modules.".format(module))
             log.info("Use the command 'nf-core modules list' to view available software")
             return False
-        log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash))
-
         # Set the install folder based on the repository name
         install_folder = ["nf-core", "software"]
         if not self.modules_repo.name == "nf-core/modules":
             install_folder = ["external"]
 
+        # Load 'modules.json'
+        modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
+        with open(modules_json_path, "r") as fh:
+            modules_json = json.load(fh)
+
+        current_version = modules_json["modules"].get(module)
+        if current_version is None:
+            if latest:
+                # Fetch the latest commit for the module
+                git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+                if len(git_log) == 0:
+                    log.error(f"Was unable to fetch version of module '{module}'")
+                    return False
+                version = git_log[0]["git_sha"]
+            else:
+                try:
+                    version = prompt_module_version_sha(module)
+                except SystemError as e:
+                    log.error(e)
+                    sys.exit(1)
+        else:
+            # Fetch the latest commit for the module
+            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+            if len(git_log) == 0:
+                log.error(f"Was unable to fetch version of module '{module}'")
+                return False
+            latest_version = git_log[0]["git_sha"]
+            if current_version == latest_version:
+                log.info("Already up to date")
+                return True
+            elif not force:
+                log.error("Found newer version of module. To install use '--force'")
+                return False
+
         # Check that we don't already have a folder for this module
         module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module)
         if os.path.exists(module_dir):
-            log.error("Module directory already exists: {}".format(module_dir))
-            # TODO: uncomment next line once update is implemented
-            # log.info("To update an existing module, use the commands 'nf-core update'")
-            return False
+            if not force:
+                log.error(
+                    "Module directory already exists but module {} is not present in 'module.json'".format(module_dir)
+                )
+                return False
+            else:
+                try:
+                    shutil.rmtree(module_dir)
+                    # Try cleaning up empty parent if tool/subtool and tool/ is empty
+                    if module.count("/") > 0:
+                        parent_dir = os.path.dirname(module_dir)
+                        try:
+                            os.rmdir(parent_dir)
+                        except OSError:
+                            log.debug(f"Parent directory not empty: '{parent_dir}'")
+                        else:
+                            log.debug(f"Deleted orphan tool directory: '{parent_dir}'")
+                    log.debug("Successfully removed {} module".format(module))
+                except OSError as e:
+                    log.error("Could not remove old version of module: {}".format(e))
+                    return False
+        # Load 'modules.json'
+        modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
+        with open(modules_json_path, "r") as fh:
+            modules_json = json.load(fh)
+
+        current_version = modules_json.get(module)
+        if current_version is None:
+            if latest or force:
+                # Fetch the latest commit for the module
+                git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+                if len(git_log) == 0:
+                    log.error(f"Was unable to fetch version of module '{module}'")
+                    return False
+                version = git_log[0]["git_sha"]
+            else:
+                try:
+                    version = prompt_module_version_sha(module)
+                except SystemError as e:
+                    log.error(e)
+                    sys.exit(1)
+        else:
+            # Fetch the latest commit for the module
+            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+            if len(git_log) == 0:
+                log.error(f"Was unable to fetch version of module '{module}'")
+                return False
+            latest_version = git_log[0]["git_sha"]
+            if current_version == latest_version:
+                log.info("Already up to date")
+                return True
+            elif not force:
+                log.error("Found newer version of module. To install use '--force'")
+                return False
+
+        log.info("Installing {}".format(module))
+
+        log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash))
 
         # Download module files
         files = self.modules_repo.get_module_file_urls(module, version)
@@ -153,9 +225,6 @@ def install(self, module=None, latest=False):
         log.info("Downloaded {} files to {}".format(len(files), module_dir))
 
         # Update module.json with new module
-        modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
-        with open(modules_json_path, "r") as fh:
-            modules_json = json.load(fh)
 
         modules_json["modules"][module] = {"git_sha": version}
         with open(modules_json_path, "w") as fh:

From 286724db43fcde807de2775281b70fe02247310f Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 13:58:32 +0200
Subject: [PATCH 25/34] Remove duplicated code

---
 nf_core/modules/pipeline_modules.py | 34 -----------------------------
 1 file changed, 34 deletions(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 5e071c645..232f2476c 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -177,40 +177,6 @@ def install(self, module=None, latest=False, force=False):
                 except OSError as e:
                     log.error("Could not remove old version of module: {}".format(e))
                     return False
-        # Load 'modules.json'
-        modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
-        with open(modules_json_path, "r") as fh:
-            modules_json = json.load(fh)
-
-        current_version = modules_json.get(module)
-        if current_version is None:
-            if latest or force:
-                # Fetch the latest commit for the module
-                git_log = get_module_git_log(module, per_page=1, page_nbr=1)
-                if len(git_log) == 0:
-                    log.error(f"Was unable to fetch version of module '{module}'")
-                    return False
-                version = git_log[0]["git_sha"]
-            else:
-                try:
-                    version = prompt_module_version_sha(module)
-                except SystemError as e:
-                    log.error(e)
-                    sys.exit(1)
-        else:
-            # Fetch the latest commit for the module
-            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
-            if len(git_log) == 0:
-                log.error(f"Was unable to fetch version of module '{module}'")
-                return False
-            latest_version = git_log[0]["git_sha"]
-            if current_version == latest_version:
-                log.info("Already up to date")
-                return True
-            elif not force:
-                log.error("Found newer version of module. To install use '--force'")
-                return False
-
         log.info("Installing {}".format(module))
 
         log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash))

From d033d3d64edb3677473dbdc8984f9e51d62bf5ee Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 14:10:10 +0200
Subject: [PATCH 26/34] Added commit SHA to message, and bug fix of '--force'

---
 nf_core/modules/module_utils.py     | 13 ++++++++---
 nf_core/modules/pipeline_modules.py | 34 ++++++++++++++---------------
 2 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 0b22b0bb5..6f5402bd3 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -158,7 +158,7 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_
     return all(files_are_equal)
 
 
-def prompt_module_version_sha(module):
+def prompt_module_version_sha(module, old_sha=None):
     older_commits_choice = questionary.Choice(
         title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value=""
     )
@@ -170,8 +170,15 @@ def prompt_module_version_sha(module):
         next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1)
         choices = []
         for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
-            tag_display = [("fg:ansiblue", f"{title}  "), ("class:choice-default", "")]
-            choices.append(questionary.Choice(title=tag_display, value=sha))
+
+            display_color = "fg:ansiblue" if sha != old_sha else "fg:ansired"
+            message = f"{title} {sha}"
+            if old_sha == sha:
+                message += " (old version)"
+            print(message)
+            print(sha, old_sha)
+            commit_display = [(display_color, message), ("class:choice-default", "")]
+            choices.append(questionary.Choice(title=commit_display, value=sha))
         if len(next_page_commits) > 0:
             choices += [older_commits_choice]
         git_sha = questionary.select(
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 232f2476c..65f6e767f 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -125,33 +125,33 @@ def install(self, module=None, latest=False, force=False):
             modules_json = json.load(fh)
 
         current_version = modules_json["modules"].get(module)
-        if current_version is None:
-            if latest:
-                # Fetch the latest commit for the module
-                git_log = get_module_git_log(module, per_page=1, page_nbr=1)
-                if len(git_log) == 0:
-                    log.error(f"Was unable to fetch version of module '{module}'")
-                    return False
-                version = git_log[0]["git_sha"]
-            else:
-                try:
-                    version = prompt_module_version_sha(module)
-                except SystemError as e:
-                    log.error(e)
-                    sys.exit(1)
-        else:
+        print(current_version)
+        if current_version is not None:
             # Fetch the latest commit for the module
             git_log = get_module_git_log(module, per_page=1, page_nbr=1)
             if len(git_log) == 0:
                 log.error(f"Was unable to fetch version of module '{module}'")
                 return False
-            latest_version = git_log[0]["git_sha"]
-            if current_version == latest_version:
+            version = git_log[0]["git_sha"]
+            if current_version == version:
                 log.info("Already up to date")
                 return True
             elif not force:
                 log.error("Found newer version of module. To install use '--force'")
                 return False
+        if latest:
+            # Fetch the latest commit for the module
+            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+            if len(git_log) == 0:
+                log.error(f"Was unable to fetch version of module '{module}'")
+                return False
+            version = git_log[0]["git_sha"]
+        else:
+            try:
+                version = prompt_module_version_sha(module, old_sha=current_version["git_sha"])
+            except SystemError as e:
+                log.error(e)
+                sys.exit(1)
 
         # Check that we don't already have a folder for this module
         module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module)

From dabdc642dd58421f72844e4c1c7e0409c3b3d7a0 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 14:28:02 +0200
Subject: [PATCH 27/34] Bug fixes for '--force'

---
 nf_core/modules/module_utils.py     | 10 +++----
 nf_core/modules/pipeline_modules.py | 41 ++++++++++++++++++-----------
 2 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py
index 6f5402bd3..6e65a8e1b 100644
--- a/nf_core/modules/module_utils.py
+++ b/nf_core/modules/module_utils.py
@@ -158,7 +158,7 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_
     return all(files_are_equal)
 
 
-def prompt_module_version_sha(module, old_sha=None):
+def prompt_module_version_sha(module, installed_sha=None):
     older_commits_choice = questionary.Choice(
         title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value=""
     )
@@ -171,12 +171,10 @@ def prompt_module_version_sha(module, old_sha=None):
         choices = []
         for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
 
-            display_color = "fg:ansiblue" if sha != old_sha else "fg:ansired"
+            display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired"
             message = f"{title} {sha}"
-            if old_sha == sha:
-                message += " (old version)"
-            print(message)
-            print(sha, old_sha)
+            if installed_sha == sha:
+                message += " (installed version)"
             commit_display = [(display_color, message), ("class:choice-default", "")]
             choices.append(questionary.Choice(title=commit_display, value=sha))
         if len(next_page_commits) > 0:
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 65f6e767f..4550cf9cd 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -125,33 +125,44 @@ def install(self, module=None, latest=False, force=False):
             modules_json = json.load(fh)
 
         current_version = modules_json["modules"].get(module)
-        print(current_version)
+
         if current_version is not None:
             # Fetch the latest commit for the module
             git_log = get_module_git_log(module, per_page=1, page_nbr=1)
             if len(git_log) == 0:
                 log.error(f"Was unable to fetch version of module '{module}'")
                 return False
-            version = git_log[0]["git_sha"]
-            if current_version == version:
+            latest_version = git_log[0]["git_sha"]
+            if current_version == latest_version:
                 log.info("Already up to date")
                 return True
             elif not force:
                 log.error("Found newer version of module. To install use '--force'")
                 return False
-        if latest:
-            # Fetch the latest commit for the module
-            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
-            if len(git_log) == 0:
-                log.error(f"Was unable to fetch version of module '{module}'")
-                return False
-            version = git_log[0]["git_sha"]
+
+            # Check for flag '--latest'
+            if latest:
+                version = latest_version
+            else:
+                try:
+                    version = prompt_module_version_sha(module, installed_sha=current_version["git_sha"])
+                except SystemError as e:
+                    log.error(e)
         else:
-            try:
-                version = prompt_module_version_sha(module, old_sha=current_version["git_sha"])
-            except SystemError as e:
-                log.error(e)
-                sys.exit(1)
+            # Check for flag '--latest'
+            if latest:
+                # Fetch the latest commit for the module
+                git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+                if len(git_log) == 0:
+                    log.error(f"Was unable to fetch version of module '{module}'")
+                    return False
+                version = git_log[0]["git_sha"]
+            else:
+                try:
+                    version = prompt_module_version_sha(module)
+                except SystemError as e:
+                    log.error(e)
+                    sys.exit(1)
 
         # Check that we don't already have a folder for this module
         module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module)

From a9a769e269cbc3fb686fadf814d4768963282cbe Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 15:15:35 +0200
Subject: [PATCH 28/34] Added '--sha' flag to specify tool version

---
 nf_core/__main__.py                 |  5 +++--
 nf_core/modules/pipeline_modules.py | 19 ++++++++++++++-----
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 6e10169f8..2b2381837 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -382,7 +382,8 @@ def list(ctx, pipeline_dir, json):
 @click.option("-t", "--tool", type=str, metavar="<tool> or <tool/subtool>")
 @click.option("-l", "--latest", is_flag=True, default=False, help="Install the latest version of the module")
 @click.option("--force", is_flag=True, default=False, help="Force installation of module if module already exists")
-def install(ctx, pipeline_dir, tool, latest, force):
+@click.option("--sha", type=str, metavar="<commit sha>", help="Install module at commit SHA")
+def install(ctx, pipeline_dir, tool, latest, force, sha):
     """
     Add a DSL2 software wrapper module to a pipeline.
 
@@ -393,7 +394,7 @@ def install(ctx, pipeline_dir, tool, latest, force):
         mods = nf_core.modules.PipelineModules()
         mods.modules_repo = ctx.obj["modules_repo_obj"]
         mods.pipeline_dir = pipeline_dir
-        mods.install(tool, latest, force)
+        mods.install(tool, latest, force, sha)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 4550cf9cd..2cbced227 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -94,7 +94,7 @@ def list_modules(self, print_json=False):
             return json.dumps(modules, sort_keys=True, indent=4)
         return table
 
-    def install(self, module=None, latest=False, force=False):
+    def install(self, module=None, latest=False, force=False, sha=""):
 
         # Check whether pipelines is valid
         self.has_valid_pipeline()
@@ -143,6 +143,8 @@ def install(self, module=None, latest=False, force=False):
             # Check for flag '--latest'
             if latest:
                 version = latest_version
+            elif sha is not None:
+                version = sha
             else:
                 try:
                     version = prompt_module_version_sha(module, installed_sha=current_version["git_sha"])
@@ -157,6 +159,8 @@ def install(self, module=None, latest=False, force=False):
                     log.error(f"Was unable to fetch version of module '{module}'")
                     return False
                 version = git_log[0]["git_sha"]
+            elif sha is not None:
+                version = sha
             else:
                 try:
                     version = prompt_module_version_sha(module)
@@ -164,6 +168,8 @@ def install(self, module=None, latest=False, force=False):
                     log.error(e)
                     sys.exit(1)
 
+        log.info("Installing {}".format(module))
+
         # Check that we don't already have a folder for this module
         module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module)
         if os.path.exists(module_dir):
@@ -173,6 +179,7 @@ def install(self, module=None, latest=False, force=False):
                 )
                 return False
             else:
+                log.info(f"Removing old version of module '{module}'")
                 try:
                     shutil.rmtree(module_dir)
                     # Try cleaning up empty parent if tool/subtool and tool/ is empty
@@ -188,7 +195,6 @@ def install(self, module=None, latest=False, force=False):
                 except OSError as e:
                     log.error("Could not remove old version of module: {}".format(e))
                     return False
-        log.info("Installing {}".format(module))
 
         log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash))
 
@@ -198,11 +204,14 @@ def install(self, module=None, latest=False, force=False):
         for filename, api_url in files.items():
             split_filename = filename.split("/")
             dl_filename = os.path.join(self.pipeline_dir, "modules", *install_folder, *split_filename[1:])
-            self.modules_repo.download_gh_file(dl_filename, api_url)
+            try:
+                self.modules_repo.download_gh_file(dl_filename, api_url)
+            except SystemError as e:
+                log.error(e)
+                return False
         log.info("Downloaded {} files to {}".format(len(files), module_dir))
 
-        # Update module.json with new module
-
+        # Update module.json with newly installed module
         modules_json["modules"][module] = {"git_sha": version}
         with open(modules_json_path, "w") as fh:
             json.dump(modules_json, fh, indent=4)

From 0c4cda5b58bb7fd9ffa632ca61a636f892194ef6 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 17:42:43 +0200
Subject: [PATCH 29/34] Refactor modules install

---
 nf_core/__main__.py                 |   5 +-
 nf_core/modules/pipeline_modules.py | 183 ++++++++++++++++------------
 2 files changed, 107 insertions(+), 81 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 2b2381837..313c80e5e 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -394,7 +394,10 @@ def install(ctx, pipeline_dir, tool, latest, force, sha):
         mods = nf_core.modules.PipelineModules()
         mods.modules_repo = ctx.obj["modules_repo_obj"]
         mods.pipeline_dir = pipeline_dir
-        mods.install(tool, latest, force, sha)
+        mods.force = force
+        mods.latest = latest
+        mods.sha = sha
+        mods.install(tool)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 2cbced227..f25a23c37 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -46,6 +46,9 @@ def __init__(self):
         self.modules_repo = ModulesRepo()
         self.pipeline_dir = None
         self.pipeline_module_names = []
+        self.force = False
+        self.latest = False
+        self.sha = None
 
     def list_modules(self, print_json=False):
         """
@@ -94,13 +97,15 @@ def list_modules(self, print_json=False):
             return json.dumps(modules, sort_keys=True, indent=4)
         return table
 
-    def install(self, module=None, latest=False, force=False, sha=""):
+    def install(self, module=None):
 
         # Check whether pipelines is valid
         self.has_valid_pipeline()
 
         # Get the available modules
         self.modules_repo.get_modules_file_tree()
+        if self.latest and self.sha is not None:
+            log.error("Cannot use '--sha' and '--latest' at the same time!")
 
         if module is None:
             module = questionary.autocomplete(
@@ -119,102 +124,79 @@ def install(self, module=None, latest=False, force=False, sha=""):
         if not self.modules_repo.name == "nf-core/modules":
             install_folder = ["external"]
 
+        # Compute the module directory
+        module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module)
+
         # Load 'modules.json'
         modules_json_path = os.path.join(self.pipeline_dir, "modules.json")
         with open(modules_json_path, "r") as fh:
             modules_json = json.load(fh)
 
-        current_version = modules_json["modules"].get(module)
+        current_entry = modules_json["modules"].get(module)
 
-        if current_version is not None:
+        if current_entry is not None and self.sha is None:
             # Fetch the latest commit for the module
+            current_version = current_entry["git_sha"]
             git_log = get_module_git_log(module, per_page=1, page_nbr=1)
             if len(git_log) == 0:
                 log.error(f"Was unable to fetch version of module '{module}'")
                 return False
             latest_version = git_log[0]["git_sha"]
-            if current_version == latest_version:
+            if current_version == latest_version and not self.force:
                 log.info("Already up to date")
                 return True
-            elif not force:
-                log.error("Found newer version of module. To install use '--force'")
-                return False
+            elif not self.force:
+                log.error("Found newer version of module.")
+                self.latest = self.force = questionary.confirm(
+                    "Do you want install it? (--force --latest)", default=False
+                ).unsafe_ask()
+                if not self.latest:
+                    return False
+        else:
+            latest_version = None
 
-            # Check for flag '--latest'
-            if latest:
-                version = latest_version
-            elif sha is not None:
-                version = sha
+        # Check that we don't already have a folder for this module
+        if not self.check_module_files_installed(module, module_dir):
+            return False
+
+        if self.sha is not None:
+            if not current_entry is None and not self.force:
+                return False
+            if self.download_module_file(module, self.sha, install_folder, module_dir):
+                self.update_modules_json(modules_json, modules_json_path, module, self.sha)
+                return True
             else:
                 try:
-                    version = prompt_module_version_sha(module, installed_sha=current_version["git_sha"])
+                    version = prompt_module_version_sha(module, installed_sha=current_entry["git_sha"])
                 except SystemError as e:
                     log.error(e)
+                    return False
         else:
-            # Check for flag '--latest'
-            if latest:
+            if self.latest:
                 # Fetch the latest commit for the module
-                git_log = get_module_git_log(module, per_page=1, page_nbr=1)
-                if len(git_log) == 0:
-                    log.error(f"Was unable to fetch version of module '{module}'")
-                    return False
-                version = git_log[0]["git_sha"]
-            elif sha is not None:
-                version = sha
+                if latest_version is None:
+                    git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+                    if len(git_log) == 0:
+                        log.error(f"Was unable to fetch version of module '{module}'")
+                        return False
+                    latest_version = git_log[0]["git_sha"]
+                version = latest_version
             else:
                 try:
                     version = prompt_module_version_sha(module)
                 except SystemError as e:
                     log.error(e)
-                    sys.exit(1)
-
-        log.info("Installing {}".format(module))
-
-        # Check that we don't already have a folder for this module
-        module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module)
-        if os.path.exists(module_dir):
-            if not force:
-                log.error(
-                    "Module directory already exists but module {} is not present in 'module.json'".format(module_dir)
-                )
-                return False
-            else:
-                log.info(f"Removing old version of module '{module}'")
-                try:
-                    shutil.rmtree(module_dir)
-                    # Try cleaning up empty parent if tool/subtool and tool/ is empty
-                    if module.count("/") > 0:
-                        parent_dir = os.path.dirname(module_dir)
-                        try:
-                            os.rmdir(parent_dir)
-                        except OSError:
-                            log.debug(f"Parent directory not empty: '{parent_dir}'")
-                        else:
-                            log.debug(f"Deleted orphan tool directory: '{parent_dir}'")
-                    log.debug("Successfully removed {} module".format(module))
-                except OSError as e:
-                    log.error("Could not remove old version of module: {}".format(e))
                     return False
 
+        log.info("Installing {}".format(module))
         log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash))
 
         # Download module files
-        files = self.modules_repo.get_module_file_urls(module, version)
-        log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys())))
-        for filename, api_url in files.items():
-            split_filename = filename.split("/")
-            dl_filename = os.path.join(self.pipeline_dir, "modules", *install_folder, *split_filename[1:])
-            try:
-                self.modules_repo.download_gh_file(dl_filename, api_url)
-            except SystemError as e:
-                log.error(e)
-                return False
-        log.info("Downloaded {} files to {}".format(len(files), module_dir))
+        if not self.download_module_file(module, version, install_folder, module_dir):
+            return False
 
         # Update module.json with newly installed module
-        modules_json["modules"][module] = {"git_sha": version}
-        with open(modules_json_path, "w") as fh:
-            json.dump(modules_json, fh, indent=4)
+        self.update_modules_json(modules_json, modules_json_path, module, version)
         return True
 
     def update(self, module, force=False):
@@ -258,22 +240,7 @@ def remove(self, module):
         log.info("Removing {}".format(module))
 
         # Remove the module
-        try:
-            shutil.rmtree(module_dir)
-            # Try cleaning up empty parent if tool/subtool and tool/ is empty
-            if module.count("/") > 0:
-                parent_dir = os.path.dirname(module_dir)
-                try:
-                    os.rmdir(parent_dir)
-                except OSError:
-                    log.debug(f"Parent directory not empty: '{parent_dir}'")
-                else:
-                    log.debug(f"Deleted orphan tool directory: '{parent_dir}'")
-            log.info("Successfully removed {} module".format(module))
-            return True
-        except OSError as e:
-            log.error("Could not remove module: {}".format(e))
-            return False
+        return self.clear_module_dir(module_name=module, module_dir=module_dir)
 
     def get_pipeline_modules(self):
         """Get list of modules installed in the current pipeline"""
@@ -302,3 +269,59 @@ def has_modules_file(self):
         if not os.path.exists(modules_json_path):
             log.info("Creating missing 'module.json' file.")
             create_modules_json(self.pipeline_dir)
+
+    def clear_module_dir(self, module_name, module_dir):
+        """Removes a module directory recursively"""
+        try:
+            shutil.rmtree(module_dir)
+            # Try cleaning up empty parent if tool/subtool and tool/ is empty
+            if module_name.count("/") > 0:
+                parent_dir = os.path.dirname(module_dir)
+                try:
+                    os.rmdir(parent_dir)
+                except OSError:
+                    log.debug(f"Parent directory not empty: '{parent_dir}'")
+                else:
+                    log.debug(f"Deleted orphan tool directory: '{parent_dir}'")
+            log.debug("Successfully removed {} module".format(module_name))
+            return True
+        except OSError as e:
+            log.error("Could not remove module: {}".format(e))
+            return False
+
+    def download_module_file(self, module_name, module_version, install_folder, module_dir):
+        """Downloads the files of a module from the remote repo"""
+        files = self.modules_repo.get_module_file_urls(module_name, module_version)
+        log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys())))
+        for filename, api_url in files.items():
+            split_filename = filename.split("/")
+            dl_filename = os.path.join(self.pipeline_dir, "modules", *install_folder, *split_filename[1:])
+            try:
+                self.modules_repo.download_gh_file(dl_filename, api_url)
+            except SystemError as e:
+                log.error(e)
+                return False
+        log.info("Downloaded {} files to {}".format(len(files), module_dir))
+        return True
+
+    def check_module_files_installed(self, module_name, module_dir):
+        """Checks if a module is already installed"""
+        if os.path.exists(module_dir):
+            if not self.force:
+                log.error(f"Module directory '{module_dir}' already exists.")
+                self.force = questionary.confirm(
+                    "Do you want to overwrite local files? (--force)", default=False
+                ).unsafe_ask()
+            if self.force:
+                log.info(f"Removing old version of module '{module_name}'")
+                return self.clear_module_dir(module_name, module_dir)
+            else:
+                return False
+        else:
+            return True
+
+    def update_modules_json(self, modules_json, modules_json_path, module_name, module_version):
+        """Updates the 'module.json' file with new module info"""
+        modules_json["modules"][module_name] = {"git_sha": module_version}
+        with open(modules_json_path, "w") as fh:
+            json.dump(modules_json, fh, indent=4)

From 89339e1b36388e5789481e37f48407ce981d34d5 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 17:49:33 +0200
Subject: [PATCH 30/34] Update tests with new flags

---
 tests/test_modules.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/tests/test_modules.py b/tests/test_modules.py
index f65e213fd..d792d297e 100644
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -40,9 +40,11 @@ def setUp(self):
         shutil.copytree(self.template_dir, self.pipeline_dir)
         self.mods = nf_core.modules.PipelineModules()
         self.mods.pipeline_dir = self.pipeline_dir
+        self.mods.latest = self.mods.force = True
         self.mods_alt = nf_core.modules.PipelineModules()
         self.mods_alt.pipeline_dir = self.pipeline_dir
         self.mods_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master")
+        self.mods_alt.latest = self.mods_alt.force = True
 
         # Set up the nf-core/modules repo dummy
         self.nfcore_modules = create_modules_repo_dummy()
@@ -65,46 +67,46 @@ def test_modules_list(self):
     def test_modules_install_nopipeline(self):
         """Test installing a module - no pipeline given"""
         self.mods.pipeline_dir = None
-        assert self.mods.install("foo", latest=True) is False
+        assert self.mods.install("foo") is False
 
     def test_modules_install_emptypipeline(self):
         """Test installing a module - empty dir given"""
         self.mods.pipeline_dir = tempfile.mkdtemp()
         with pytest.raises(UserWarning) as excinfo:
-            self.mods.install("foo", latest=True)
+            self.mods.install("foo")
         assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value)
 
     def test_modules_install_nomodule(self):
         """Test installing a module - unrecognised module given"""
-        assert self.mods.install("foo", latest=True) is False
+        assert self.mods.install("foo") is False
 
     def test_modules_install_trimgalore(self):
         """Test installing a module - TrimGalore!"""
-        assert self.mods.install("trimgalore", latest=True) is not False
+        assert self.mods.install("trimgalore") is not False
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "trimgalore")
         assert os.path.exists(module_path)
 
     def test_modules_install_trimgalore_alternative_source(self):
         """Test installing a module from a different source repository - TrimGalore!"""
-        assert self.mods_alt.install("trimgalore", latest=True) is not False
+        assert self.mods_alt.install("trimgalore") is not False
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "external", "trimgalore")
         assert os.path.exists(module_path)
 
     def test_modules_install_trimgalore_twice(self):
         """Test installing a module - TrimGalore! already there"""
-        self.mods.install("trimgalore", latest=True)
-        assert self.mods.install("trimgalore", latest=True) is False
+        self.mods.install("trimgalore")
+        assert self.mods.install("trimgalore") is True
 
     def test_modules_remove_trimgalore(self):
         """Test removing TrimGalore! module after installing it"""
-        self.mods.install("trimgalore", latest=True)
+        self.mods.install("trimgalore")
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "trimgalore")
         assert self.mods.remove("trimgalore")
         assert os.path.exists(module_path) is False
 
     def test_modules_remove_trimgalore_alternative_source(self):
         """Test removing TrimGalore! module after installing it from an alternative source"""
-        self.mods_alt.install("trimgalore", latest=True)
+        self.mods_alt.install("trimgalore")
         module_path = os.path.join(self.mods.pipeline_dir, "modules", "external", "trimgalore")
         assert self.mods_alt.remove("trimgalore")
         assert os.path.exists(module_path) is False
@@ -115,7 +117,7 @@ def test_modules_remove_trimgalore_uninstalled(self):
 
     def test_modules_lint_trimgalore(self):
         """Test linting the TrimGalore! module"""
-        self.mods.install("trimgalore", latest=True)
+        self.mods.install("trimgalore")
         module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir)
         module_lint.lint(print_results=False, module="trimgalore")
         assert len(module_lint.passed) == 20

From 50876190b73ab7b889620923161cfc8b9ea47082 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Tue, 22 Jun 2021 18:09:27 +0200
Subject: [PATCH 31/34] Make questionary select show installed version

---
 nf_core/modules/pipeline_modules.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index f25a23c37..72fee2b8e 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -183,7 +183,9 @@ def install(self, module=None):
                 version = latest_version
             else:
                 try:
-                    version = prompt_module_version_sha(module)
+                    version = prompt_module_version_sha(
+                        module, installed_sha=current_entry["git_sha"] if not current_entry is None else None
+                    )
                 except SystemError as e:
                     log.error(e)
                     return False
@@ -271,7 +273,6 @@ def has_modules_file(self):
             create_modules_json(self.pipeline_dir)
 
     def clear_module_dir(self, module_name, module_dir):
-        """Removes a module directory recursively"""
         try:
             shutil.rmtree(module_dir)
             # Try cleaning up empty parent if tool/subtool and tool/ is empty

From e39ff09582b4308192e032ed25b6ff1b9543b36f Mon Sep 17 00:00:00 2001
From: Erik Danielsson <53212377+ErikDanielsson@users.noreply.github.com>
Date: Wed, 23 Jun 2021 09:16:56 +0200
Subject: [PATCH 32/34] Apply suggestions from code review

Co-authored-by: Kevin Menden <kevin.menden@live.com>
---
 nf_core/__main__.py                 | 4 ++--
 nf_core/modules/pipeline_modules.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 313c80e5e..22eca0eba 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -381,8 +381,8 @@ def list(ctx, pipeline_dir, json):
 @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.option("-t", "--tool", type=str, metavar="<tool> or <tool/subtool>")
 @click.option("-l", "--latest", is_flag=True, default=False, help="Install the latest version of the module")
-@click.option("--force", is_flag=True, default=False, help="Force installation of module if module already exists")
-@click.option("--sha", type=str, metavar="<commit sha>", help="Install module at commit SHA")
+@click.option("-f", "--force", is_flag=True, default=False, help="Force installation of module if module already exists")
+@click.option("-s", "--sha", type=str, metavar="<commit sha>", help="Install module at commit SHA")
 def install(ctx, pipeline_dir, tool, latest, force, sha):
     """
     Add a DSL2 software wrapper module to a pipeline.
diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 72fee2b8e..a03ee1632 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -159,7 +159,7 @@ def install(self, module=None):
         if not self.check_module_files_installed(module, module_dir):
             return False
 
-        if self.sha is not None:
+        if self.sha:
             if not current_entry is None and not self.force:
                 return False
             if self.download_module_file(module, self.sha, install_folder, module_dir):

From 2586097a076a48b9d8bfb27d9b9d7a8c6eba456a Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Wed, 23 Jun 2021 09:18:14 +0200
Subject: [PATCH 33/34] Apply changes from code review

---
 nf_core/modules/pipeline_modules.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py
index 72fee2b8e..38ff6ded5 100644
--- a/nf_core/modules/pipeline_modules.py
+++ b/nf_core/modules/pipeline_modules.py
@@ -106,6 +106,7 @@ def install(self, module=None):
         self.modules_repo.get_modules_file_tree()
         if self.latest and self.sha is not None:
             log.error("Cannot use '--sha' and '--latest' at the same time!")
+            return False
 
         if module is None:
             module = questionary.autocomplete(

From 5b7c79be623d4085ec4747139d8d116460edaa26 Mon Sep 17 00:00:00 2001
From: Erik Danielsson <danielsson.erik.0@gmail.com>
Date: Wed, 23 Jun 2021 09:26:52 +0200
Subject: [PATCH 34/34] Black

---
 nf_core/__main__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 22eca0eba..8db50fab7 100755
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -381,7 +381,9 @@ def list(ctx, pipeline_dir, json):
 @click.argument("pipeline_dir", type=click.Path(exists=True), required=True, metavar="<pipeline directory>")
 @click.option("-t", "--tool", type=str, metavar="<tool> or <tool/subtool>")
 @click.option("-l", "--latest", is_flag=True, default=False, help="Install the latest version of the module")
-@click.option("-f", "--force", is_flag=True, default=False, help="Force installation of module if module already exists")
+@click.option(
+    "-f", "--force", is_flag=True, default=False, help="Force installation of module if module already exists"
+)
 @click.option("-s", "--sha", type=str, metavar="<commit sha>", help="Install module at commit SHA")
 def install(ctx, pipeline_dir, tool, latest, force, sha):
     """