add support for --list-software

easybuild/framework/easyconfig/easyconfig.py

@@ -924,7 +924,10 @@ def _generate_template_values(self, ignore=None, skip_lower=True):
         # (eg the run_setp code in EasyBlock)
 
         # step 1-3 work with easyconfig.templates constants
-        template_values = template_constant_dict(self._config, ignore=ignore, skip_lower=skip_lower)
+        # disable templating with creating dict with template values to avoid looping back to here via __getitem__
+        self.enable_templating = False
+        template_values = template_constant_dict(self, ignore=ignore, skip_lower=skip_lower)
+        self.enable_templating = True
 
         # update the template_values dict
         self.template_values.update(template_values)
@@ -1199,6 +1202,7 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
     :param path: path to easyconfig file
     :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
     :param validate: whether or not to perform validation
+    :param parse_only: only parse easyconfig superficially (faster, but results in partial info)
     :param hidden: indicate whether corresponding module file should be installed hidden ('.'-prefixed)
     """
     blocks = retrieve_blocks_in_spec(path, build_option('only_blocks'))

easybuild/framework/easyconfig/templates.py

@@ -160,7 +160,7 @@ def template_constant_dict(config, ignore=None, skip_lower=True):
             continue
 
         if name[0].startswith('toolchain_'):
-            tc = config.get('toolchain')[0]
+            tc = config.get('toolchain')
             if tc is not None:
                 template_values['toolchain_name'] = tc.get('name', None)
                 template_values['toolchain_version'] = tc.get('version', None)
@@ -169,7 +169,7 @@ def template_constant_dict(config, ignore=None, skip_lower=True):
 
         elif name[0].startswith('version_'):
             # parse major and minor version numbers
-            version = config['version'][0]
+            version = config['version']
             if version is not None:
 
                 _log.debug("version found in easyconfig is %s", version)
@@ -189,27 +189,34 @@ def template_constant_dict(config, ignore=None, skip_lower=True):
         elif name[0].endswith('letter'):
             # parse first letters
             if name[0].startswith('name'):
-                softname = config['name'][0]
+                softname = config['name']
                 if softname is not None:
                     template_values['nameletter'] = softname[0]
         else:
             raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
 
     # step 2: define *ver and *shortver templates
     for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
-        for dep in config['dependencies'][0]:
-            if isinstance(dep['name'], basestring) and dep['name'].lower() == name.lower():
-                template_values['%sver' % pref] = dep['version']
-                template_values['%sshortver' % pref] = '.'.join(dep['version'].split('.')[:2])
+        for dep in config.get('dependencies', []):
+            if isinstance(dep, dict):
+                dep_name, dep_version = dep['name'], dep['version']
+            elif isinstance(dep, (list, tuple)):
+                dep_name, dep_version = dep[0], dep[1]
+            else:
+                raise EasyBuildError("Unexpected type for dependency: %s", dep)
+
+            if isinstance(dep_name, basestring) and dep_name.lower() == name.lower():
+                template_values['%sver' % pref] = dep_version
+                template_values['%sshortver' % pref] = '.'.join(dep_version.split('.')[:2])
                 break
 
     # step 3: add remaining from config
     for name in TEMPLATE_NAMES_CONFIG:
         if name in ignore:
             continue
         if name in config:
-            template_values[name] = config[name][0]
-            _log.debug('name: %s, config: %s', name, config[name][0])
+            template_values[name] = config[name]
+            _log.debug('name: %s, config: %s', name, config[name])
 
     # step 4. make lower variants if not skip_lower
     if not skip_lower:

easybuild/main.py

@@ -55,6 +55,7 @@
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
+from easybuild.tools.docs import list_software
 from easybuild.tools.filetools import adjust_permissions, cleanup, write_file
 from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token, new_pr, update_pr
 from easybuild.tools.modules import modules_tool
@@ -253,8 +254,18 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     elif options.review_pr:
         print review_pr(options.review_pr, colored=use_color(options.color))
 
+    elif options.list_software:
+        print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')
+
     # non-verbose cleanup after handling GitHub integration stuff or printing terse info
-    if options.check_github or options.install_github_token or options.review_pr or options.terse:
+    early_stop_options = [
+        options.check_github,
+        options.install_github_token,
+        options.list_software,
+        options.review_pr,
+        options.terse,
+    ]
+    if any(early_stop_options):
         cleanup(logfile, eb_tmpdir, testing, silent=True)
         sys.exit(0)
 

easybuild/tools/docs.py

@@ -38,7 +38,9 @@
 import inspect
 import os
 import re
+import string
 import sys
+from distutils.version import LooseVersion
 from vsc.utils import fancylogger
 from vsc.utils.docs import mk_rst_table
 from vsc.utils.missing import nub
@@ -48,14 +50,18 @@
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
 from easybuild.framework.easyconfig.easyconfig import get_easyblock_class
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
+from easybuild.framework.easyconfig.parser import EasyConfigParser
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_EASYCONFIG
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_CONSTANTS
-from easybuild.framework.easyconfig.templates import TEMPLATE_SOFTWARE_VERSIONS
+from easybuild.framework.easyconfig.templates import TEMPLATE_SOFTWARE_VERSIONS, template_constant_dict
+from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs
 from easybuild.framework.extension import Extension
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.config import build_option
 from easybuild.tools.filetools import read_file
 from easybuild.tools.ordereddict import OrderedDict
+from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
 from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
 from easybuild.tools.utilities import import_available_modules, quote_str
 
@@ -81,11 +87,13 @@ def generate_doc(name, params):
 
 def rst_title_and_table(title, table_titles, table_values):
     """Generate table in section with title in .rst format."""
-    doc = [
-        title,
-        '-' * len(title),
-        '',
-    ]
+    doc = []
+    if title is not None:
+        doc.extend([
+            title,
+            '-' * len(title),
+            '',
+        ])
     doc.extend(mk_rst_table(table_titles, table_values))
     return doc
 
@@ -514,6 +522,178 @@ def add_class(classes, cls):
     return '\n'.join(txt)
 
 
+def list_software(output_format=FORMAT_TXT, detailed=False):
+    """
+    Show list of supported software
+
+    :param output_format: output format to use
+    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+    :return: multi-line string presenting requested info
+    """
+    silent = build_option('silent')
+
+    ec_paths = find_matching_easyconfigs('*', '*', build_option('robot_path') or [])
+    ecs = []
+    cnt = len(ec_paths)
+    for idx, ec_path in enumerate(ec_paths):
+        ecs.append(EasyConfigParser(filename=ec_path).get_config_dict())
+        print_msg('\r', prefix=False, newline=False, silent=silent)
+        print_msg("Processed %d/%d easyconfigs..." % (idx+1, cnt), newline=False, silent=silent)
+    print_msg('', prefix=False, silent=silent)
+
+    software = {}
+    for ec in ecs:
+        software.setdefault(ec['name'], [])
+        if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME:
+            toolchain = DUMMY_TOOLCHAIN_NAME
+        else:
+            toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version'])
+
+        template_values = template_constant_dict(ec)
+
+        software[ec['name']].append({
+            'description': ec['description'],
+            'homepage': ec['homepage'],
+            'toolchain': toolchain,
+            'version': ec['version'],
+            'versionsuffix': ec.get('versionsuffix', '') % template_values,
+        })
+
+    print_msg("Found %d different software packages" % len(software), silent=silent)
+
+    return generate_doc('list_software_%s' % output_format, [software, detailed])
+
+
+def list_software_rst(software, detailed=False):
+    """
+    Return overview of supported software in RST format
+
+    :param software: software information (structured like list_software does)
+    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+    :return: multi-line string presenting requested info
+    """
+
+    title = "List of supported software"
+    lines = [
+        title,
+        '=' * len(title),
+        '',
+        "EasyBuild |version| supports %d different software packages (incl. toolchains, bundles):" % len(software),
+        '',
+    ]
+
+    # links to per-letter tables
+    letter_refs = ''
+    key_letters = nub(sorted(k[0].lower() for k in software.keys()))
+    for letter in string.lowercase:
+        if letter in key_letters:
+            if letter_refs:
+                letter_refs += " - :ref:`list_software_letter_%s`" % letter
+            else:
+                letter_refs = ":ref:`list_software_letter_%s`" % letter
+    lines.extend([letter_refs, ''])
+
+    def key_to_ref(name):
+        """Create a reference label for the specified software name."""
+        return 'list_software_%s_%d' % (name, sum(ord(l) for l in name))
+
+    letter = None
+    sorted_keys = sorted(software.keys(), key=lambda x: x.lower())
+    for key in sorted_keys:
+
+        # start a new subsection for each letter
+        if key[0].lower() != letter:
+
+            # subsection for new letter
+            letter = key[0].lower()
+            lines.extend([
+                '',
+                '.. _list_software_letter_%s:' % letter,
+                '',
+                "*%s*" % letter.upper(),
+                '-' * 3,
+                '',
+            ])
+
+            if detailed:
+                # quick links per software package
+                lines.extend([
+                    '',
+                    ' - '.join(':ref:`%s`' % key_to_ref(k) for k in sorted_keys if k[0].lower() == letter),
+                    '',
+                ])
+
+        # append software to list, including version(suffix) & toolchain info if detailed info is requested
+        if detailed:
+            table_titles = ['version', 'toolchain']
+            table_values = [[], []]
+
+            pairs = nub((x['version'], x['versionsuffix']) for x in software[key])
+
+            with_vsuff = any(vs for (_, vs) in pairs)
+            if with_vsuff:
+                table_titles.insert(1, 'versionsuffix')
+                table_values.insert(1, [])
+
+            for ver, vsuff in sorted((LooseVersion(v), vs) for (v, vs) in pairs):
+                table_values[0].append('``%s``' % ver)
+                if vsuff:
+                    table_values[1].append('``%s``' % vsuff)
+                tcs = [x['toolchain'] for x in software[key] if x['version'] == ver and x['versionsuffix'] == vsuff]
+                table_values[-1].append(', '.join('``%s``' % tc for tc in sorted(nub(tcs))))
+
+            lines.extend([
+                '',
+                '.. _%s:' % key_to_ref(key),
+                '',
+                '*%s*' % key,
+                '+' * (len(key) + 2),
+                '',
+                ' '.join(software[key][-1]['description'].split('\n')).lstrip(' '),
+                '',
+                "*homepage*: %s" % software[key][-1]['homepage'],
+                '',
+            ] + rst_title_and_table(None, table_titles, table_values))
+        else:
+            lines.append("* %s" % key)
+
+    return '\n'.join(lines)
+
+
+def list_software_txt(software, detailed=False):
+    """
+    Return overview of supported software in plain text
+
+    :param software: software information (structured like list_software does)
+    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+    :return: multi-line string presenting requested info
+    """
+
+    lines = ['']
+    for key in sorted(software, key=lambda x: x.lower()):
+        lines.append('* %s' % key)
+        if detailed:
+            lines.extend([
+                '',
+                ' '.join(software[key][-1]['description'].split('\n')),
+                '',
+                "homepage: %s" % software[key][-1]['homepage'],
+                '',
+            ])
+            pairs = nub((x['version'], x['versionsuffix']) for x in software[key])
+            for ver, vsuff in sorted((LooseVersion(v), vs) for (v, vs) in pairs):
+                tcs = [x['toolchain'] for x in software[key] if x['version'] == ver and x['versionsuffix'] == vsuff]
+
+                line = "  * %s v%s" % (key, ver)
+                if vsuff:
+                    line += " (versionsuffix: '%s')" % vsuff
+                line += ": %s" % ', '.join(sorted(nub(tcs)))
+                lines.append(line)
+            lines.append('')
+
+    return '\n'.join(lines)
+
+
 def list_toolchains(output_format=FORMAT_TXT):
     """Show list of known toolchains."""
     _, all_tcs = search_toolchain('')

easybuild/tools/options.py

@@ -453,6 +453,8 @@ def informative_options(self):
             'last-log': ("Print location to EasyBuild log file of last (failed) session", None, 'store_true', False),
             'list-easyblocks': ("Show list of available easyblocks",
                                 'choice', 'store_or_None', 'simple', ['simple', 'detailed']),
+            'list-software': ("Show list of supported software", 'choice', 'store_or_None', 'simple',
+                              ['simple', 'detailed']),
             'list-toolchains': ("Show list of known toolchains",
                                 None, 'store_true', False),
             'search': ("Search for easyconfig files in the robot search path, print full paths",