Allow hiding toolchains (REVIEW)

easybuild/framework/easyconfig/easyconfig.py

@@ -1249,9 +1249,9 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
 
             # add toolchain as dependency too
             if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
-                dep = ec.toolchain.as_dict()
-                _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
-                easyconfig['dependencies'].append(dep)
+                tc = ec.toolchain.as_dict()
+                _log.debug("Adding toolchain %s as dependency for app %s." % (tc, name))
+                easyconfig['dependencies'].append(tc)
 
     if cache_key is not None:
         _easyconfigs_cache[cache_key] = [e.copy() for e in easyconfigs]

easybuild/framework/easyconfig/types.py

@@ -30,6 +30,7 @@
 @author: Kenneth Hoste (Ghent University)
 """
 from vsc.utils import fancylogger
+from distutils.util import strtobool
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
@@ -255,14 +256,16 @@ def convert_value_type(val, typ):
     return res
 
 
-def to_name_version_dict(spec):
+def to_toolchain_dict(spec):
     """
-    Convert a comma-separated string or 2-element list of strings to a dictionary with name/version keys.
-    If the specified value is a dict already, the keys are checked to be only name/version.
+    Convert a comma-separated string or 2/3-element list of strings to a dictionary with name/version keys, and
+    optionally a hidden key. If the specified value is a dict already, the keys are checked to be only
+    name/version/hidden.
 
     For example: "intel, 2015a" => {'name': 'intel', 'version': '2015a'}
+                 "foss, 2016a, True" => {'name': 'foss', 'version': '2016a', 'hidden': True}
 
-    @param spec: a comma-separated string with two values, or a 2-element list of strings, or a dict
+    @param spec: a comma-separated string with two or three values, or a 2/3-element list of strings, or a dict
     """
     # check if spec is a string or a list of two values; else, it can not be converted
     if isinstance(spec, basestring):
@@ -272,22 +275,33 @@ def to_name_version_dict(spec):
         # 2-element list
         if len(spec) == 2:
             res = {'name': spec[0].strip(), 'version': spec[1].strip()}
+        # 3-element list
+        elif len(spec) == 3:
+            res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': strtobool(spec[2].strip())}
         else:
-            raise EasyBuildError("Can not convert list %s to name and version dict. Expected 2 elements", spec)
+            raise EasyBuildError("Can not convert list %s to toolchain dict. Expected 2 or 3 elements", spec)
 
     elif isinstance(spec, dict):
         # already a dict, check keys
-        if sorted(spec.keys()) == ['name', 'version']:
+        sorted_keys = sorted(spec.keys())
+        if sorted_keys == ['name', 'version'] or sorted_keys == ['hidden', 'name', 'version']:
             res = spec
         else:
-            raise EasyBuildError("Incorrect set of keys in provided dictionary, should be only name/version: %s", spec)
+            raise EasyBuildError("Incorrect set of keys in provided dictionary, should be only name/version/hidden: %s",
+                                 spec)
 
     else:
-        raise EasyBuildError("Conversion of %s (type %s) to name and version dict is not supported", spec, type(spec))
+        raise EasyBuildError("Conversion of %s (type %s) to toolchain dict is not supported", spec, type(spec))
 
     return res
 
 
+def to_name_version_dict(spec):
+    """Deprecated in favor of to_toolchain_dict."""
+    _log.deprecated("to_name_version_dict; use to_toolchain_dict instead.", '2.9')
+    return to_toolchain_dict(spec)
+
+
 def to_list_of_strings_and_tuples(spec):
     """
     Convert a 'list of lists and strings' to a 'list of tuples and strings'
@@ -370,7 +384,7 @@ def to_dependency(dep):
                 if key in ['name', 'version', 'versionsuffix']:
                     depspec[key] = str(value)
                 elif key == 'toolchain':
-                    depspec['toolchain'] = to_name_version_dict(value)
+                    depspec['toolchain'] = to_toolchain_dict(value)
                 elif not found_name_version:
                     depspec.update({'name': key, 'version': str(value)})
                 else:
@@ -422,19 +436,24 @@ def to_checksums(checksums):
 
 
 # these constants use functions defined in this module, so they needs to be at the bottom of the module
-# specific type: dict with only name/version as keys, and with string values
+# specific type: dict with only name/version as keys with string values, and optionally a hidden key with bool value
 # additional type requirements are specified as tuple of tuples rather than a dict, since this needs to be hashable
-NAME_VERSION_DICT = (dict, as_hashable({
-    'elem_types': [str],
-    'opt_keys': [],
+TOOLCHAIN_DICT = (dict, as_hashable({
+    'elem_types': {
+        'hidden': [bool],
+        'name': [str],
+        'version': [str],
+    },
+    'opt_keys': ['hidden'],
     'req_keys': ['name', 'version'],
 }))
+NAME_VERSION_DICT = TOOLCHAIN_DICT    # *DEPRECATED* in favor of TOOLCHAIN_DICT
 DEPENDENCY_DICT = (dict, as_hashable({
     'elem_types': {
         'full_mod_name': [str],
         'name': [str],
         'short_mod_name': [str],
-        'toolchain': [NAME_VERSION_DICT],
+        'toolchain': [TOOLCHAIN_DICT],
         'version': [str],
         'versionsuffix': [str],
     },
@@ -455,7 +474,7 @@ def to_checksums(checksums):
 }))
 CHECKSUMS = (list, as_hashable({'elem_types': [STRING_OR_TUPLE_LIST]}))
 
-CHECKABLE_TYPES = [CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, NAME_VERSION_DICT, SANITY_CHECK_PATHS_DICT,
+CHECKABLE_TYPES = [CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, TOOLCHAIN_DICT, SANITY_CHECK_PATHS_DICT,
                   STRING_OR_TUPLE_LIST, TUPLE_OF_STRINGS]
 
 # easy types, that can be verified with isinstance
@@ -468,7 +487,7 @@ def to_checksums(checksums):
     'osdependencies': STRING_OR_TUPLE_LIST,
     'patches': STRING_OR_TUPLE_LIST,
     'sanity_check_paths': SANITY_CHECK_PATHS_DICT,
-    'toolchain': NAME_VERSION_DICT,
+    'toolchain': TOOLCHAIN_DICT,
     'version': basestring,
 }
 # add all dependency types as dependencies
@@ -482,7 +501,7 @@ def to_checksums(checksums):
     str: str,
     CHECKSUMS: to_checksums,
     DEPENDENCIES: to_dependencies,
-    NAME_VERSION_DICT: to_name_version_dict,
+    TOOLCHAIN_DICT: to_toolchain_dict,
     SANITY_CHECK_PATHS_DICT: to_sanity_check_paths_dict,
     STRING_OR_TUPLE_LIST: to_list_of_strings_and_tuples,
 }

easybuild/tools/config.py

@@ -96,6 +96,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'extra_modules',
         'filter_deps',
         'hide_deps',
+        'hide_toolchains',
         'from_pr',
         'git_working_dirs_path',
         'pr_branch_name',

easybuild/tools/options.py

@@ -286,6 +286,8 @@ def override_options(self):
                             'strlist', 'extend', None),
             'hide-deps': ("Comma separated list of dependencies that you want automatically hidden, "
                           "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None),
+            'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, "
+                                "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None),
             'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False),
             'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS),
                             None, 'store_true', False),

easybuild/tools/toolchain/toolchain.py

@@ -79,7 +79,8 @@ def _is_toolchain_for(cls, name):
 
     _is_toolchain_for = classmethod(_is_toolchain_for)
 
-    def __init__(self, name=None, version=None, mns=None, class_constants=None, tcdeps=None, modtool=None):
+    def __init__(self, name=None, version=None, mns=None, class_constants=None, tcdeps=None, modtool=None,
+                 hidden=False):
         """
         Toolchain constructor.
 
@@ -89,6 +90,7 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde
         @param class_constants: toolchain 'constants' to define
         @param tcdeps: list of toolchain 'dependencies' (i.e., the toolchain components)
         @param modtool: ModulesTool instance to use
+        @param hidden: bool indicating whether toolchain is hidden or not
         """
 
         self.base_init()
@@ -117,6 +119,8 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde
 
         # toolchain instances are created before initiating build options sometimes, e.g. for --list-toolchains
         self.dry_run = build_option('extended_dry_run', default=False)
+        hidden_toolchains = build_option('hide_toolchains', default=None) or []
+        self.hidden = hidden or (name in hidden_toolchains)
 
         self.modules_tool = modtool
 
@@ -286,7 +290,7 @@ def as_dict(self, name=None, version=None):
             'versionsuffix': '',
             'dummy': True,
             'parsed': True,  # pretend this is a parsed easyconfig file, as may be required by det_short_module_name
-            'hidden': False,
+            'hidden': self.hidden,
             'full_mod_name': self.mod_full_name,
             'short_mod_name': self.mod_short_name,
         }

easybuild/tools/toolchain/utilities.py

@@ -132,7 +132,9 @@ def get_toolchain(tc, tcopts, mns=None, tcdeps=None, modtool=None):
         if not tc_class:
             all_tcs_names = ','.join([x.NAME for x in all_tcs])
             raise EasyBuildError("Toolchain %s not found, available toolchains: %s", tc['name'], all_tcs_names)
-        tc_inst = tc_class(version=tc['version'], mns=mns, tcdeps=tcdeps, modtool=modtool)
+
+        hidden = tc.get('hidden', False)
+        tc_inst = tc_class(version=tc['version'], mns=mns, tcdeps=tcdeps, modtool=modtool, hidden=hidden)
         tc_dict = tc_inst.as_dict()
         _log.debug("Obtained new toolchain instance for %s: %s" % (key, tc_dict))
 

test/framework/easyconfig.py

@@ -1802,6 +1802,25 @@ def test_parse_deps_templates(self):
         dep_full_mod_names = [d['full_mod_name'] for d in ordered_ecs[-1]['ec']._config['dependencies'][0]]
         self.assertEqual(dep_full_mod_names, expected)
 
+    def test_hidden_toolchain(self):
+        """Test hiding of toolchain via easyconfig parameter."""
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
+        ec_txt = read_file(os.path.join(test_ecs_dir, 'gzip-1.6-GCC-4.9.2.eb'))
+
+        new_tc = "toolchain = {'name': 'GCC', 'version': '4.9.2', 'hidden': True}"
+        ec_txt = re.sub("toolchain = .*", new_tc, ec_txt, re.M)
+
+        ec_file = os.path.join(self.test_prefix, 'test.eb')
+        write_file(ec_file, ec_txt)
+
+        args = [
+            ec_file,
+            '--dry-run',
+        ]
+        outtxt = self.eb_main(args)
+        self.assertTrue(re.search('module: GCC/\.4\.9\.2', outtxt))
+        self.assertTrue(re.search('module: gzip/1\.6-GCC-4\.9\.2', outtxt))
+
 
 def suite():
     """ returns all the testcases in this module """

test/framework/easyconfigs/yeb/bzip-bad-toolchain.yeb

@@ -13,8 +13,8 @@ description: |
     compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical
     compressors), whilst being around twice as fast at compression and six times faster at decompression.
 
-# bad toolchain with three parameters
-toolchain: GCC, 4.9, 2
+# bad toolchain with four parameters
+toolchain: GCC, 4.9, False, 2
 toolchainopts: {pic: True}
 
 sources:

test/framework/options.py

@@ -1491,7 +1491,7 @@ def test_hide_deps(self):
         # clear log file
         open(self.logfile, 'w').write('')
 
-        # filter deps (including a non-existing dep, i.e. zlib)
+        # hide deps (including a non-existing dep, i.e. zlib)
         args.append('--hide-deps=FFTW,ScaLAPACK,zlib')
         outtxt = self.eb_main(args, do_build=True, verbose=True, raise_error=True)
         self.assertTrue(re.search('module: GCC/4.7.2', outtxt))
@@ -1504,6 +1504,19 @@ def test_hide_deps(self):
         # zlib is not a dep at all
         self.assertFalse(re.search(r'module: zlib', outtxt))
 
+    def test_hide_toolchains(self):
+        """Test use of --hide-toolchains."""
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
+        ec_file = os.path.join(test_ecs_dir, 'gzip-1.6-GCC-4.9.2.eb')
+        args = [
+            ec_file,
+            '--dry-run',
+            '--hide-toolchains=GCC',
+        ]
+        outtxt = self.eb_main(args)
+        self.assertTrue(re.search('module: GCC/\.4\.9\.2', outtxt))
+        self.assertTrue(re.search('module: gzip/1\.6-GCC-4\.9\.2', outtxt))
+
     def test_test_report_env_filter(self):
         """Test use of --test-report-env-filter."""
 

test/framework/type_checking.py

@@ -33,10 +33,10 @@
 
 from easybuild.framework.easyconfig.types import as_hashable, check_element_types, check_key_types, check_known_keys
 from easybuild.framework.easyconfig.types import check_required_keys, check_type_of_param_value, convert_value_type
-from easybuild.framework.easyconfig.types import DEPENDENCIES, DEPENDENCY_DICT, NAME_VERSION_DICT
+from easybuild.framework.easyconfig.types import DEPENDENCIES, DEPENDENCY_DICT, TOOLCHAIN_DICT
 from easybuild.framework.easyconfig.types import SANITY_CHECK_PATHS_DICT, STRING_OR_TUPLE_LIST
 from easybuild.framework.easyconfig.types import is_value_of_type, to_checksums, to_dependencies, to_dependency
-from easybuild.framework.easyconfig.types import to_list_of_strings_and_tuples, to_name_version_dict
+from easybuild.framework.easyconfig.types import to_list_of_strings_and_tuples, to_toolchain_dict
 from easybuild.framework.easyconfig.types import to_sanity_check_paths_dict
 from easybuild.tools.build_log import EasyBuildError
 
@@ -195,27 +195,44 @@ class Foo():
             pass
         self.assertErrorRegex(EasyBuildError, "No conversion function available", convert_value_type, None, Foo)
 
-    def test_to_name_version_dict(self):
+    def test_to_toolchain_dict(self):
         """ Test toolchain string to dict conversion """
         # normal cases
-        self.assertEqual(to_name_version_dict("intel, 2015a"), {'name': 'intel', 'version': '2015a'})
-        self.assertEqual(to_name_version_dict(('intel', '2015a')), {'name': 'intel', 'version': '2015a'})
-        self.assertEqual(to_name_version_dict(['gcc', '4.7']), {'name': 'gcc', 'version': '4.7'})
+        self.assertEqual(to_toolchain_dict(('intel', '2015a')), {'name': 'intel', 'version': '2015a'})
+        self.assertEqual(to_toolchain_dict("intel, 2015a"), {'name': 'intel', 'version': '2015a'})
+        self.assertEqual(to_toolchain_dict(['gcc', '4.7']), {'name': 'gcc', 'version': '4.7'})
+
+        # incl. hidden spec
+        expected = {'name': 'intel', 'version': '2015a', 'hidden': True}
+        self.assertEqual(to_toolchain_dict("intel, 2015a, True"), expected)
+        expected = {'name': 'intel', 'version': '2015a', 'hidden': False}
+        self.assertEqual(to_toolchain_dict(('intel', '2015a', 'False')), expected)
+        expected = {'name': 'gcc', 'version': '4.7', 'hidden': True}
+        self.assertEqual(to_toolchain_dict(['gcc', '4.7', 'True']), expected)
+
         tc = {'name': 'intel', 'version': '2015a'}
-        self.assertEqual(to_name_version_dict(tc), tc)
+        self.assertEqual(to_toolchain_dict(tc), tc)
+
+        tc = {'name': 'intel', 'version': '2015a', 'hidden': True}
+        self.assertEqual(to_toolchain_dict(tc), tc)
 
         # wrong type
-        self.assertErrorRegex(EasyBuildError, r"Conversion of .* \(type .*\) to name and version dict is not supported",
-                              to_name_version_dict, 1000)
+        self.assertErrorRegex(EasyBuildError, r"Conversion of .* \(type .*\) to toolchain dict is not supported",
+                              to_toolchain_dict, 1000)
 
         # wrong number of elements
-        errstr = "Can not convert .* to name and version .*. Expected 2 elements"
-        self.assertErrorRegex(EasyBuildError, errstr, to_name_version_dict, "intel, 2015, a")
-        self.assertErrorRegex(EasyBuildError, errstr, to_name_version_dict, "intel")
-        self.assertErrorRegex(EasyBuildError, errstr, to_name_version_dict, ['gcc', '4', '7'])
+        errstr = "Can not convert .* to toolchain dict. Expected 2 or 3 elements"
+        self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, "intel, 2015, True, a")
+        self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, "intel")
+        self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', 'False', '7'])
+
+        # invalid truth value
+        errstr = "invalid truth value .*"
+        self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, "intel, 2015, foo")
+        self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, ['gcc', '4', '7'])
 
         # missing keys
-        self.assertErrorRegex(EasyBuildError, "Incorrect set of keys", to_name_version_dict, {'name': 'intel'})
+        self.assertErrorRegex(EasyBuildError, "Incorrect set of keys", to_toolchain_dict, {'name': 'intel'})
 
     def test_to_dependency(self):
         """ Test dependency dict to tuple conversion """
@@ -315,13 +332,13 @@ def test_is_value_of_type(self):
         self.assertFalse(is_value_of_type("foo", int))
 
         # toolchain type check
-        self.assertTrue(is_value_of_type({'name': 'intel', 'version': '2015a'}, NAME_VERSION_DICT))
+        self.assertTrue(is_value_of_type({'name': 'intel', 'version': '2015a'}, TOOLCHAIN_DICT))
         # version value should be string, not int
-        self.assertFalse(is_value_of_type({'name': 'intel', 'version': 100}, NAME_VERSION_DICT))
+        self.assertFalse(is_value_of_type({'name': 'intel', 'version': 100}, TOOLCHAIN_DICT))
         # missing version key
-        self.assertFalse(is_value_of_type({'name': 'intel', 'foo': 'bar'}, NAME_VERSION_DICT))
+        self.assertFalse(is_value_of_type({'name': 'intel', 'foo': 'bar'}, TOOLCHAIN_DICT))
         # extra key, shouldn't be there
-        self.assertFalse(is_value_of_type({'name': 'intel', 'version': '2015a', 'foo': 'bar'}, NAME_VERSION_DICT))
+        self.assertFalse(is_value_of_type({'name': 'intel', 'version': '2015a', 'foo': 'bar'}, TOOLCHAIN_DICT))
 
         # dependency type check
         self.assertTrue(is_value_of_type({'name': 'intel', 'version': '2015a'}, DEPENDENCY_DICT))
@@ -332,7 +349,7 @@ def test_is_value_of_type(self):
             'versionsuffix': 'foo',
         }, DEPENDENCY_DICT))
         # no version key
-        self.assertFalse(is_value_of_type({'name': 'intel'}, NAME_VERSION_DICT))
+        self.assertFalse(is_value_of_type({'name': 'intel'}, TOOLCHAIN_DICT))
         # too many keys
         self.assertFalse(is_value_of_type({
             'name': 'intel',

test/framework/yeb.py

@@ -155,7 +155,7 @@ def test_bad_toolchain_format(self):
         # only test bad cases - the right ones are tested with the test files in test_parse_yeb
         testdir = os.path.dirname(os.path.abspath(__file__))
         test_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
-        expected = r'Can not convert list .* to name and version dict. Expected 2 elements'
+        expected = r'Can not convert list .* to toolchain dict. Expected 2 or 3 elements'
         self.assertErrorRegex(EasyBuildError, expected, EasyConfig, os.path.join(test_easyconfigs, 'bzip-bad-toolchain.yeb'))
 
     def test_external_module_toolchain(self):