diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 3c4467ad3c..48ee994d62 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -387,12 +387,12 @@ def init(options, config_options_dict): def init_build_options(build_options=None, cmdline_options=None): """Initialize build options.""" - # building a dependency graph implies force, so that all dependencies are retained - # and also skips validation of easyconfigs (e.g. checking os dependencies) active_build_options = {} if cmdline_options is not None: + # building a dependency graph implies force, so that all dependencies are retained + # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if cmdline_options.dep_graph: _log.info("Enabling force to generate dependency graph.") diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 1098f59f1a..c2ffc8c74a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -75,18 +75,32 @@ class EasyBuildOptions(GeneralOption): ALLOPTSMANDATORY = False # allow more than one argument + def __init__(self, *args, **kwargs): + """Constructor.""" + + self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] + + # set up constants to seed into config files parser, by section + self.go_cfg_constants = { + self.DEFAULTSECT: { + 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), + "List of default robot paths ('%s'-separated)" % os.pathsep), + } + } + + # update or define go_configfiles_initenv in named arguments to pass to parent constructor + go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) + for section, constants in self.go_cfg_constants.items(): + constants = dict([(name, value) for (name, (value, _)) in constants.items()]) + go_cfg_initenv.setdefault(section, {}).update(constants) + + super(EasyBuildOptions, self).__init__(*args, **kwargs) + def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] - easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) - if easyconfigs_pkg_paths: - default_robot_paths = easyconfigs_pkg_paths - else: - self.log.warning("basic_options: unable to determine easyconfigs pkg path for --robot-paths default") - default_robot_paths = [] - descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ @@ -100,7 +114,7 @@ def basic_options(self): 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", - 'pathlist', 'store', default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'pathlist', 'store', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), @@ -256,6 +270,8 @@ def informative_options(self): descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ + 'avail-cfgfile-constants': ("Show all constants that can be used in configuration files", + None, 'store_true', False), 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", @@ -369,7 +385,7 @@ def postprocess(self): # prepare for --list/--avail if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, - self.options.list_easyblocks, self.options.list_toolchains, + self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, @@ -423,6 +439,11 @@ def _postprocess_config(self): def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' + + # dump supported configuration file constants + if self.options.avail_cfgfile_constants: + msg += self.avail_cfgfile_constants() + # dump possible easyconfig params if self.options.avail_easyconfig_params: msg += self.avail_easyconfig_params() @@ -469,6 +490,23 @@ def _postprocess_list_avail(self): print msg sys.exit(0) + def avail_cfgfile_constants(self): + """ + Return overview of constants supported in configuration files. + """ + lines = [ + "Constants available (only) in configuration files:", + "syntax: %(CONSTANT_NAME)s", + ] + for section in self.go_cfg_constants: + lines.append('') + if section != self.DEFAULTSECT: + section_title = "only in '%s' section:" % section + lines.append(section_title) + for cst_name, (cst_value, cst_help) in sorted(self.go_cfg_constants[section].items()): + lines.append("* %s: %s [value: %s]" % (cst_name, cst_help, cst_value)) + return '\n'.join(lines) + def avail_easyconfig_params(self): """ Print the available easyconfig parameters, for the given easyblock. diff --git a/test/framework/config.py b/test/framework/config.py index b28ec141cf..fb4cdd854b 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -31,6 +31,7 @@ import copy import os import shutil +import sys import tempfile from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader @@ -42,7 +43,7 @@ from easybuild.tools.config import log_file_format, set_tmpdir, BuildOptions, ConfigurationVariables from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options, build_option from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import write_file +from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.repository.repository import init_repository @@ -443,10 +444,22 @@ def test_generaloption_config_file(self): self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + # test with config file passed via environment variable cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, + 'robot-paths = /tmp/foo:%(DEFAULT_ROBOT_PATHS)s', ]) write_file(config_file, cfgtxt) @@ -460,6 +473,8 @@ def test_generaloption_config_file(self): self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file + self.assertTrue('/tmp/foo' in options.robot_paths) + self.assertTrue(os.path.join(tmpdir, 'easybuild', 'easyconfigs') in options.robot_paths) testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 @@ -474,6 +489,7 @@ def test_generaloption_config_file(self): self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES'] + sys.path[:] = orig_sys_path def test_set_tmpdir(self): """Test set_tmpdir config function.""" diff --git a/test/framework/options.py b/test/framework/options.py index 0ed83a5e74..17f3688640 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -453,6 +453,39 @@ def test_avail_lists(self): if os.path.exists(dummylogfn): os.remove(dummylogfn) + def test_avail_cfgfile_constants(self): + """Test --avail-cfgfile-constants.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + + args = [ + '--avail-cfgfile-constants', + '--unittest-file=%s' % self.logfile, + ] + outtxt = self.eb_main(args, logfile=dummylogfn) + cfgfile_constants = { + 'DEFAULT_ROBOT_PATHS': os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + } + for cst_name, cst_value in cfgfile_constants.items(): + cst_regex = re.compile("^\*\s%s:\s.*\s\[value: .*%s.*\]" % (cst_name, cst_value), re.M) + tup = (cst_regex.pattern, outtxt) + self.assertTrue(cst_regex.search(outtxt), "Pattern '%s' in --avail-cfgfile_constants output: %s" % tup) + + if os.path.exists(dummylogfn): + os.remove(dummylogfn) + sys.path[:] = orig_sys_path + def test_list_easyblocks(self): """Test listing easyblock hierarchy.""" diff --git a/vsc/README.md b/vsc/README.md index 932369c661..b5efeb45be 100644 --- a/vsc/README.md +++ b/vsc/README.md @@ -1,3 +1,3 @@ Code from https://github.com/hpcugent/vsc-base -based on 35fee9d3130a6b52bf83993e73d187e9d46c69bc (vsc-base v1.9.9) +based on 2146be5301da34043adf4646169e5dfec88cd2f5 (vsc-base v1.9.9) diff --git a/vsc/utils/generaloption.py b/vsc/utils/generaloption.py index 390e5fa79e..e4548dc668 100644 --- a/vsc/utils/generaloption.py +++ b/vsc/utils/generaloption.py @@ -699,6 +699,7 @@ class GeneralOption(object): VERSION = None # set the version (will add --version) + DEFAULTSECT = ConfigParser.DEFAULTSECT DEFAULT_LOGLEVEL = None DEFAULT_CONFIGFILES = None DEFAULT_IGNORECONFIGFILES = None @@ -1073,7 +1074,7 @@ def configfile_parser_init(self, initenv=None): for name, section in initenv.items(): name = str(name) - if name == ConfigParser.DEFAULTSECT: + if name == self.DEFAULTSECT: # is protected/reserved (and hidden) pass elif not self.configfile_parser.has_section(name): @@ -1128,7 +1129,7 @@ def parseconfigfiles(self): self.log.debug("parseconfigfiles: following files were NOT parsed %s" % [x for x in configfiles if not x in parsed_files]) self.log.debug("parseconfigfiles: sections (w/o %s) %s" % - (ConfigParser.DEFAULTSECT, self.configfile_parser.sections())) + (self.DEFAULTSECT, self.configfile_parser.sections())) # walk through list of section names # - look for options set though config files @@ -1176,7 +1177,7 @@ def parseconfigfiles(self): actual_option = self.parser.get_option_by_long_name(opt_name) if actual_option is None: # don't fail on DEFAULT UPPERCASE options in case-sensitive mode. - in_def = self.configfile_parser.has_option(ConfigParser.DEFAULTSECT, opt) + in_def = self.configfile_parser.has_option(self.DEFAULTSECT, opt) if in_def and self.CONFIGFILE_CASESENSITIVE and opt == opt.upper(): self.log.debug(('parseconfigfiles: no option corresponding with ' 'opt %s dest %s in section %s but found all uppercase '