Merge pull request #1100 from boegel/robot_path_cst

support use of %(DEFAULT_ROBOT_PATHS)s template in EasyBuild configuration files

easybuild/tools/config.py

@@ -387,12 +387,12 @@ def init(options, config_options_dict):
 
 def init_build_options(build_options=None, cmdline_options=None):
     """Initialize build options."""
-    # building a dependency graph implies force, so that all dependencies are retained
-    # and also skips validation of easyconfigs (e.g. checking os dependencies)
 
     active_build_options = {}
 
     if cmdline_options is not None:
+        # building a dependency graph implies force, so that all dependencies are retained
+        # and also skips validation of easyconfigs (e.g. checking os dependencies)
         retain_all_deps = False
         if cmdline_options.dep_graph:
             _log.info("Enabling force to generate dependency graph.")

easybuild/tools/options.py

@@ -75,18 +75,32 @@ class EasyBuildOptions(GeneralOption):
 
     ALLOPTSMANDATORY = False  # allow more than one argument
 
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+
+        self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or []
+
+        # set up constants to seed into config files parser, by section
+        self.go_cfg_constants = {
+            self.DEFAULTSECT: {
+                'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths),
+                                        "List of default robot paths ('%s'-separated)" % os.pathsep),
+            }
+        }
+
+        # update or define go_configfiles_initenv in named arguments to pass to parent constructor
+        go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {})
+        for section, constants in self.go_cfg_constants.items():
+            constants = dict([(name, value) for (name, (value, _)) in constants.items()])
+            go_cfg_initenv.setdefault(section, {}).update(constants)
+
+        super(EasyBuildOptions, self).__init__(*args, **kwargs)
+
     def basic_options(self):
         """basic runtime options"""
         all_stops = [x[0] for x in EasyBlock.get_steps()]
         strictness_options = [run.IGNORE, run.WARN, run.ERROR]
 
-        easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None)
-        if easyconfigs_pkg_paths:
-            default_robot_paths = easyconfigs_pkg_paths
-        else:
-            self.log.warning("basic_options: unable to determine easyconfigs pkg path for --robot-paths default")
-            default_robot_paths = []
-
         descr = ("Basic options", "Basic runtime options for EasyBuild.")
 
         opts = OrderedDict({
@@ -100,7 +114,7 @@ def basic_options(self):
             'robot': ("Enable dependency resolution, using easyconfigs in specified paths",
                       'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}),
             'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)",
-                            'pathlist', 'store', default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}),
+                            'pathlist', 'store', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}),
             'skip': ("Skip existing software (useful for installing additional packages)",
                      None, 'store_true', False, 'k'),
             'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops),
@@ -256,6 +270,8 @@ def informative_options(self):
         descr = ("Informative options", "Obtain information about EasyBuild.")
 
         opts = OrderedDict({
+            'avail-cfgfile-constants': ("Show all constants that can be used in configuration files",
+                                        None, 'store_true', False),
             'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs",
                                            None, 'store_true', False),
             'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs",
@@ -369,7 +385,7 @@ def postprocess(self):
 
         # prepare for --list/--avail
         if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
-                self.options.list_easyblocks, self.options.list_toolchains,
+                self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants,
                 self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
                 self.options.avail_repositories, self.options.show_default_moduleclasses,
                 self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
@@ -423,6 +439,11 @@ def _postprocess_config(self):
     def _postprocess_list_avail(self):
         """Create all the additional info that can be requested (exit at the end)"""
         msg = ''
+
+        # dump supported configuration file constants
+        if self.options.avail_cfgfile_constants:
+            msg += self.avail_cfgfile_constants()
+
         # dump possible easyconfig params
         if self.options.avail_easyconfig_params:
             msg += self.avail_easyconfig_params()
@@ -469,6 +490,23 @@ def _postprocess_list_avail(self):
             print msg
         sys.exit(0)
 
+    def avail_cfgfile_constants(self):
+        """
+        Return overview of constants supported in configuration files.
+        """
+        lines = [
+            "Constants available (only) in configuration files:",
+            "syntax: %(CONSTANT_NAME)s",
+        ]
+        for section in self.go_cfg_constants:
+            lines.append('')
+            if section != self.DEFAULTSECT:
+                section_title = "only in '%s' section:" % section
+                lines.append(section_title)
+            for cst_name, (cst_value, cst_help) in sorted(self.go_cfg_constants[section].items()):
+                lines.append("* %s: %s [value: %s]" % (cst_name, cst_help, cst_value))
+        return '\n'.join(lines)
+
     def avail_easyconfig_params(self):
         """
         Print the available easyconfig parameters, for the given easyblock.

test/framework/config.py

@@ -31,6 +31,7 @@
 import copy
 import os
 import shutil
+import sys
 import tempfile
 from test.framework.utilities import EnhancedTestCase, init_config
 from unittest import TestLoader
@@ -42,7 +43,7 @@
 from easybuild.tools.config import log_file_format, set_tmpdir, BuildOptions, ConfigurationVariables
 from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options, build_option
 from easybuild.tools.environment import modify_env
-from easybuild.tools.filetools import write_file
+from easybuild.tools.filetools import mkdir, write_file
 from easybuild.tools.repository.filerepo import FileRepository
 from easybuild.tools.repository.repository import init_repository
 
@@ -443,10 +444,22 @@ def test_generaloption_config_file(self):
         self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')])  # default
         self.assertEqual(install_path(), os.path.join(testpath2, 'software'))  # via config file
 
+        # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
+        # to check whether easyconfigs install path is auto-included in robot path
+        tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path')
+        mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
+
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
+        shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
+
+        orig_sys_path = sys.path[:]
+        sys.path.insert(0, tmpdir)  # prepend to give it preference over possible other installed easyconfigs pkgs
+
         # test with config file passed via environment variable
         cfgtxt = '\n'.join([
             '[config]',
             'buildpath = %s' % testpath1,
+            'robot-paths = /tmp/foo:%(DEFAULT_ROBOT_PATHS)s',
         ])
         write_file(config_file, cfgtxt)
 
@@ -460,6 +473,8 @@ def test_generaloption_config_file(self):
         self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software'))  # default
         self.assertEqual(source_paths(), [testpath2])  # via command line
         self.assertEqual(build_path(), testpath1)  # via config file
+        self.assertTrue('/tmp/foo' in options.robot_paths)
+        self.assertTrue(os.path.join(tmpdir, 'easybuild', 'easyconfigs') in options.robot_paths)
 
         testpath3 = os.path.join(self.tmpdir, 'testTHREE')
         os.environ['EASYBUILD_SOURCEPATH'] = testpath2
@@ -474,6 +489,7 @@ def test_generaloption_config_file(self):
         self.assertEqual(build_path(), testpath1)  # via config file
 
         del os.environ['EASYBUILD_CONFIGFILES']
+        sys.path[:] = orig_sys_path
 
     def test_set_tmpdir(self):
         """Test set_tmpdir config function."""

test/framework/options.py

@@ -453,6 +453,39 @@ def test_avail_lists(self):
         if os.path.exists(dummylogfn):
             os.remove(dummylogfn)
 
+    def test_avail_cfgfile_constants(self):
+        """Test --avail-cfgfile-constants."""
+        fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
+        os.close(fd)
+
+        # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
+        # to check whether easyconfigs install path is auto-included in robot path
+        tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path')
+        mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
+
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
+        shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
+
+        orig_sys_path = sys.path[:]
+        sys.path.insert(0, tmpdir)  # prepend to give it preference over possible other installed easyconfigs pkgs
+
+        args = [
+            '--avail-cfgfile-constants',
+            '--unittest-file=%s' % self.logfile,
+        ]
+        outtxt = self.eb_main(args, logfile=dummylogfn)
+        cfgfile_constants = {
+            'DEFAULT_ROBOT_PATHS': os.path.join(tmpdir, 'easybuild', 'easyconfigs'),
+        }
+        for cst_name, cst_value in cfgfile_constants.items():
+            cst_regex = re.compile("^\*\s%s:\s.*\s\[value: .*%s.*\]" % (cst_name, cst_value), re.M)
+            tup = (cst_regex.pattern, outtxt)
+            self.assertTrue(cst_regex.search(outtxt), "Pattern '%s' in --avail-cfgfile_constants output: %s" % tup)
+
+        if os.path.exists(dummylogfn):
+            os.remove(dummylogfn)
+        sys.path[:] = orig_sys_path
+
     def test_list_easyblocks(self):
         """Test listing easyblock hierarchy."""
 

vsc/README.md

@@ -1,3 +1,3 @@
 Code from https://github.com/hpcugent/vsc-base
 
-based on 35fee9d3130a6b52bf83993e73d187e9d46c69bc (vsc-base v1.9.9)
+based on 2146be5301da34043adf4646169e5dfec88cd2f5 (vsc-base v1.9.9)

vsc/utils/generaloption.py

@@ -699,6 +699,7 @@ class GeneralOption(object):
 
     VERSION = None  # set the version (will add --version)
 
+    DEFAULTSECT = ConfigParser.DEFAULTSECT
     DEFAULT_LOGLEVEL = None
     DEFAULT_CONFIGFILES = None
     DEFAULT_IGNORECONFIGFILES = None
@@ -1073,7 +1074,7 @@ def configfile_parser_init(self, initenv=None):
 
         for name, section in initenv.items():
             name = str(name)
-            if name == ConfigParser.DEFAULTSECT:
+            if name == self.DEFAULTSECT:
                 # is protected/reserved (and hidden)
                 pass
             elif not self.configfile_parser.has_section(name):
@@ -1128,7 +1129,7 @@ def parseconfigfiles(self):
         self.log.debug("parseconfigfiles: following files were NOT parsed %s" %
                        [x for x in configfiles if not x in parsed_files])
         self.log.debug("parseconfigfiles: sections (w/o %s) %s" %
-                       (ConfigParser.DEFAULTSECT, self.configfile_parser.sections()))
+                       (self.DEFAULTSECT, self.configfile_parser.sections()))
 
         # walk through list of section names
         # - look for options set though config files
@@ -1176,7 +1177,7 @@ def parseconfigfiles(self):
                     actual_option = self.parser.get_option_by_long_name(opt_name)
                     if actual_option is None:
                         # don't fail on DEFAULT UPPERCASE options in case-sensitive mode.
-                        in_def = self.configfile_parser.has_option(ConfigParser.DEFAULTSECT, opt)
+                        in_def = self.configfile_parser.has_option(self.DEFAULTSECT, opt)
                         if in_def and self.CONFIGFILE_CASESENSITIVE and opt == opt.upper():
                             self.log.debug(('parseconfigfiles: no option corresponding with '
                                             'opt %s dest %s in section %s but found all uppercase '