add support for --list-installed-software

easybuild/main.py

@@ -254,13 +254,18 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     elif options.review_pr:
         print review_pr(options.review_pr, colored=use_color(options.color))
 
+    elif options.list_installed_software:
+        detailed = options.list_installed_software == 'detailed'
+        print list_software(output_format=options.output_format, detailed=detailed, only_installed=True)
+
     elif options.list_software:
         print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')
 
     # non-verbose cleanup after handling GitHub integration stuff or printing terse info
     early_stop_options = [
         options.check_github,
         options.install_github_token,
+        options.list_installed_software,
         options.list_software,
         options.review_pr,
         options.terse,

easybuild/tools/docs.py

@@ -48,7 +48,7 @@
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG, HIDDEN, sorted_categories
 from easybuild.framework.easyblock import EasyBlock
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
-from easybuild.framework.easyconfig.easyconfig import get_easyblock_class
+from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, process_easyconfig
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
 from easybuild.framework.easyconfig.parser import EasyConfigParser
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_EASYCONFIG
@@ -60,9 +60,10 @@
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import read_file
+from easybuild.tools.modules import modules_tool
 from easybuild.tools.ordereddict import OrderedDict
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
-from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
+from easybuild.tools.toolchain.utilities import search_toolchain
 from easybuild.tools.utilities import import_available_modules, quote_str
 
 
@@ -522,12 +523,13 @@ def add_class(classes, cls):
     return '\n'.join(txt)
 
 
-def list_software(output_format=FORMAT_TXT, detailed=False):
+def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
     """
     Show list of supported software
 
     :param output_format: output format to use
     :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+    :param only_installed: only retain software for which a corresponding module is available
     :return: multi-line string presenting requested info
     """
     silent = build_option('silent')
@@ -536,7 +538,14 @@ def list_software(output_format=FORMAT_TXT, detailed=False):
     ecs = []
     cnt = len(ec_paths)
     for idx, ec_path in enumerate(ec_paths):
-        ecs.append(EasyConfigParser(filename=ec_path).get_config_dict())
+        # full EasyConfig instance is only required when module name is needed
+        # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
+        if only_installed:
+            ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]['ec']
+        else:
+            ec = EasyConfigParser(filename=ec_path).get_config_dict()
+
+        ecs.append(ec)
         print_msg('\r', prefix=False, newline=False, silent=silent)
         print_msg("Processed %d/%d easyconfigs..." % (idx+1, cnt), newline=False, silent=silent)
     print_msg('', prefix=False, silent=silent)
@@ -549,18 +558,38 @@ def list_software(output_format=FORMAT_TXT, detailed=False):
         else:
             toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version'])
 
-        template_values = template_constant_dict(ec)
+        versionsuffix = ec.get('versionsuffix', '')
+
+        # make sure versionsuffix gets properly templated
+        if versionsuffix and isinstance(ec, dict):
+            template_values = template_constant_dict(ec)
+            versionsuffix = versionsuffix % template_values
 
         software[ec['name']].append({
             'description': ec['description'],
             'homepage': ec['homepage'],
             'toolchain': toolchain,
             'version': ec['version'],
-            'versionsuffix': ec.get('versionsuffix', '') % template_values,
+            'versionsuffix': versionsuffix,
         })
 
+        if only_installed:
+            software[ec['name']][-1].update({'mod_name': ec.full_mod_name})
+
     print_msg("Found %d different software packages" % len(software), silent=silent)
 
+    if only_installed:
+        avail_mod_names = modules_tool().available()
+
+        # rebuild software, only retain entries with a corresponding available module
+        software, all_software = {}, software
+        for key in all_software:
+            for entry in all_software[key]:
+                if entry['mod_name'] in avail_mod_names:
+                    software.setdefault(key, []).append(entry)
+
+        print_msg("Retained %d installed software packages" % len(software), silent=silent)
+
     return generate_doc('list_software_%s' % output_format, [software, detailed])
 
 

easybuild/tools/options.py

@@ -456,6 +456,8 @@ def informative_options(self):
             'last-log': ("Print location to EasyBuild log file of last (failed) session", None, 'store_true', False),
             'list-easyblocks': ("Show list of available easyblocks",
                                 'choice', 'store_or_None', 'simple', ['simple', 'detailed']),
+            'list-installed-software': ("Show list of installed software", 'choice', 'store_or_None', 'simple',
+                                        ['simple', 'detailed']),
             'list-software': ("Show list of supported software", 'choice', 'store_or_None', 'simple',
                               ['simple', 'detailed']),
             'list-toolchains': ("Show list of known toolchains",

test/framework/docs.py

@@ -208,6 +208,19 @@ def test_list_software(self):
         txt = list_software(output_format='rst', detailed=True)
         self.assertTrue(expected.match(txt), "Pattern '%s' found in: %s" % (expected.pattern, txt))
 
+        # GCC/4.6.3 is installed, no gzip module installed
+        txt = list_software(output_format='txt', detailed=True, only_installed=True)
+        self.assertTrue(re.search('^\* GCC', txt, re.M))
+        self.assertTrue(re.search('^\s*\* GCC v4.6.3: dummy', txt, re.M))
+        self.assertFalse(re.search('^\* gzip', txt, re.M))
+        self.assertFalse(re.search('gzip v1\.', txt, re.M))
+
+        txt = list_software(output_format='rst', detailed=True, only_installed=True)
+        self.assertTrue(re.search('^\*GCC\*', txt, re.M))
+        self.assertTrue(re.search('4\.6\.3.*dummy', txt, re.M))
+        self.assertFalse(re.search('^\*gzip\*', txt, re.M))
+        self.assertFalse(re.search('1\.4', txt, re.M))
+        self.assertFalse(re.search('1\.5', txt, re.M))
 
 
 def suite():

test/framework/options.py

@@ -2799,6 +2799,69 @@ def test_use_color(self):
         easybuild.tools.options.terminal_supports_colors = lambda _: False
         self.assertFalse(use_color('auto'))
 
+    def test_list_software(self):
+        """Test --list-software and --list-installed-software."""
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0')
+        args = [
+            '--list-software',
+            '--robot-paths=%s' % test_ecs,
+        ]
+        self.mock_stdout(True)
+        self.eb_main(args, testing=False)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+        expected = '\n'.join([
+            "== Processed 5/5 easyconfigs... ",
+            "== Found 2 different software packages",
+            '',
+            "* GCC",
+            "* gzip",
+            '',
+        ])
+        self.assertTrue(txt.endswith(expected))
+
+        args = [
+            '--list-software=detailed',
+            '--output-format=rst',
+            '--robot-paths=%s' % test_ecs,
+        ]
+        self.mock_stdout(True)
+        self.eb_main(args, testing=False)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+        self.assertTrue(re.search('^\*GCC\*', txt, re.M))
+        self.assertTrue(re.search('^``4.6.3``\s+``dummy``', txt, re.M))
+        self.assertTrue(re.search('^\*gzip\*', txt, re.M))
+        self.assertTrue(re.search('^``1.5``\s+``goolf/1.4.10``,\s+``ictce/4.1.13``', txt, re.M))
+
+        args = [
+            '--list-installed-software',
+            '--output-format=rst',
+            '--robot-paths=%s' % test_ecs,
+        ]
+        self.mock_stdout(True)
+        self.eb_main(args, testing=False)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+        self.assertTrue(re.search('== Processed 5/5 easyconfigs...', txt, re.M))
+        self.assertTrue(re.search('== Found 2 different software packages', txt, re.M))
+        self.assertTrue(re.search('== Retained 1 installed software packages', txt, re.M))
+        self.assertTrue(re.search('^\* GCC', txt, re.M))
+        self.assertFalse(re.search('gzip', txt, re.M))
+
+        args = [
+            '--list-installed-software=detailed',
+            '--robot-paths=%s' % test_ecs,
+        ]
+        self.mock_stdout(True)
+        self.eb_main(args, testing=False)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+        self.assertTrue(re.search('^== Retained 1 installed software packages', txt, re.M))
+        self.assertTrue(re.search('^\* GCC', txt, re.M))
+        self.assertTrue(re.search('^\s+\* GCC v4.6.3: dummy', txt, re.M))
+        self.assertFalse(re.search('gzip', txt, re.M))
+
 
 def suite():
     """ returns all the testcases in this module """