Merge pull request #2562 from boegel/parse_hook

add 'parse' hook to add support for tweaking 'raw' easyconfig (REVIEW)

easybuild/framework/easyblock.py

@@ -76,7 +76,8 @@
 from easybuild.tools.filetools import write_file
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP
-from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP, run_hook
+from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
+from easybuild.tools.hooks import load_hooks, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -123,7 +124,7 @@ def extra_options(extra=None):
     #
     # INIT
     #
-    def __init__(self, ec, hooks=None):
+    def __init__(self, ec):
         """
         Initialize the EasyBlock instance.
         :param ec: a parsed easyconfig file (EasyConfig instance)
@@ -133,7 +134,7 @@ def __init__(self, ec, hooks=None):
         self.orig_workdir = os.getcwd()
 
         # list of pre- and post-step hooks
-        self.hooks = hooks or []
+        self.hooks = load_hooks(build_option('hooks'))
 
         # list of patch/source files, along with checksums
         self.patches = []
@@ -2766,7 +2767,7 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
-def build_and_install_one(ecdict, init_env, hooks=None):
+def build_and_install_one(ecdict, init_env):
     """
     Build the software
     :param ecdict: dictionary contaning parsed easyconfig + metadata
@@ -2804,7 +2805,7 @@ def build_and_install_one(ecdict, init_env, hooks=None):
     try:
         app_class = get_easyblock_class(easyblock, name=name)
 
-        app = app_class(ecdict['ec'], hooks=hooks)
+        app = app_class(ecdict['ec'])
         _log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
     except EasyBuildError, err:
         print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),

easybuild/framework/easyconfig/easyconfig.py

@@ -60,6 +60,7 @@
 from easybuild.tools.config import build_option, get_module_naming_scheme
 from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX
 from easybuild.tools.filetools import copy_file, decode_class_name, encode_class_name, read_file, write_file
+from easybuild.tools.hooks import PARSE, load_hooks, run_hook
 from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
@@ -475,22 +476,35 @@ def parse(self):
         # we need toolchain to be set when we call _parse_dependency
         for key in ['toolchain'] + local_vars.keys():
             # validations are skipped, just set in the config
-            # do not store variables we don't need
             if key in self._config.keys():
-                if key in ['dependencies']:
-                    self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
-                elif key in ['builddependencies']:
-                    self[key] = [self._parse_dependency(dep, build_only=True) for dep in local_vars[key]]
-                elif key in ['hiddendependencies']:
-                    self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]]
-                else:
-                    self[key] = local_vars[key]
+                self[key] = local_vars[key]
                 self.log.info("setting config option %s: value %s (type: %s)", key, self[key], type(self[key]))
             elif key in REPLACED_PARAMETERS:
                 _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')
 
+            # do not store variables we don't need
             else:
-                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))
+                self.log.debug("Ignoring unknown easyconfig parameter %s (value: %s)" % (key, local_vars[key]))
+
+        # trigger parse hook
+        # templating is disabled when parse_hook is called to allow for easy updating of mutable easyconfig parameters
+        # (see also comment in resolve_template)
+        hooks = load_hooks(build_option('hooks'))
+        prev_enable_templating = self.enable_templating
+        self.enable_templating = False
+
+        parse_hook_msg = None
+        if self.path:
+            parse_hook_msg = "Running %s hook for %s..." % (PARSE, os.path.basename(self.path))
+
+        run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg)
+        self.enable_templating = prev_enable_templating
+
+        # parse dependency specifications
+        self.log.info("Parsing dependency specifications...")
+        self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']]
+        self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']]
+        self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']]
 
         # update templating dictionary
         self.generate_template_values()

easybuild/framework/easyconfig/tools.py

@@ -369,6 +369,7 @@ def parse_easyconfigs(paths, validate=True):
     """
     easyconfigs = []
     generated_ecs = False
+
     for (path, generated) in paths:
         path = os.path.abspath(path)
         # keep track of whether any files were generated
@@ -378,12 +379,13 @@ def parse_easyconfigs(paths, validate=True):
         try:
             ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
             for ec_file in ec_files:
-                # only pass build specs when not generating easyconfig files
                 kwargs = {'validate': validate}
+                # only pass build specs when not generating easyconfig files
                 if not build_option('try_to_generate'):
                     kwargs['build_specs'] = build_option('build_specs')
-                ecs = process_easyconfig(ec_file, **kwargs)
-                easyconfigs.extend(ecs)
+
+                easyconfigs.extend(process_easyconfig(ec_file, **kwargs))
+
         except IOError, err:
             raise EasyBuildError("Processing easyconfigs in path %s failed: %s", path, err)
 

easybuild/main.py

@@ -106,27 +106,24 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True, hooks=None):
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     """
     Build and install software for all provided parsed easyconfig files.
 
     :param ecs: easyconfig files to install software with
     :param init_session_state: initial session state, to use in test reports
     :param exit_on_failure: whether or not to exit on installation failure
-    :param hooks: list of defined pre- and post-step hooks
     """
     # obtain a copy of the starting environment so each build can start afresh
     # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
-    run_hook(START, hooks)
-
     res = []
     for ec in ecs:
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, hooks=hooks)
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
@@ -165,8 +162,6 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, ho
 
         res.append((ec, ec_res))
 
-    run_hook(END, hooks)
-
     return res
 
 
@@ -284,6 +279,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     config.init(options, config_options_dict)
     config.init_build_options(build_options=build_options, cmdline_options=options)
 
+    # load hook implementations (if any)
+    hooks = load_hooks(options.hooks)
+
+    run_hook(START, hooks)
+
     if modtool is None:
         modtool = modules_tool(testing=testing)
 
@@ -492,10 +492,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     # build software, will exit when errors occurs (except when testing)
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
-        hooks = load_hooks(options.hooks)
 
-        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
-                                                  exit_on_failure=exit_on_failure, hooks=hooks)
+        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 
@@ -518,6 +516,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         if 'original_spec' in ec and os.path.isfile(ec['spec']):
             os.remove(ec['spec'])
 
+    run_hook(END, hooks)
+
     # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
     stop_logging(logfile, logtostdout=options.logtostdout)
     if overall_success:

easybuild/tools/config.py

@@ -142,6 +142,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'github_user',
         'github_org',
         'group',
+        'hooks',
         'ignore_dirs',
         'job_backend_config',
         'job_cores',

easybuild/tools/hooks.py

@@ -56,6 +56,7 @@
 TESTCASES_STEP = 'testcases'
 
 START = 'start'
+PARSE = 'parse'
 END = 'end'
 
 PRE_PREF = 'pre_'
@@ -67,39 +68,51 @@
               INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
               PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
 
-KNOWN_HOOKS = [h + HOOK_SUFF for h in [START] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]]
+HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
+KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]
+
+
+# cached version of hooks, to avoid having to load them from file multiple times
+_cached_hooks = {}
 
 
 def load_hooks(hooks_path):
     """Load defined hooks (if any)."""
-    hooks = {}
-
-    if hooks_path:
-        if not os.path.exists(hooks_path):
-            raise EasyBuildError("Specified path for hooks implementation does not exist: %s", hooks_path)
-
-        (hooks_filename, hooks_file_ext) = os.path.splitext(os.path.split(hooks_path)[1])
-        if hooks_file_ext == '.py':
-            _log.info("Importing hooks implementation from %s...", hooks_path)
-            try:
-                # import module that defines hooks, and collect all functions of which name ends with '_hook'
-                imported_hooks = imp.load_source(hooks_filename, hooks_path)
-                for attr in dir(imported_hooks):
-                    if attr.endswith(HOOK_SUFF):
-                        hook = getattr(imported_hooks, attr)
-                        if callable(hook):
-                            hooks.update({attr: hook})
-                        else:
-                            _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
-                _log.info("Found hooks: %s", sorted(hooks.keys()))
-            except ImportError as err:
-                raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err)
-        else:
-            raise EasyBuildError("Provided path for hooks implementation should be location of a Python file (*.py)")
+
+    if hooks_path in _cached_hooks:
+        hooks = _cached_hooks[hooks_path]
+
     else:
-        _log.info("No location for hooks implementation provided, no hooks defined")
+        hooks = {}
+        if hooks_path:
+            if not os.path.exists(hooks_path):
+                raise EasyBuildError("Specified path for hooks implementation does not exist: %s", hooks_path)
+
+            (hooks_filename, hooks_file_ext) = os.path.splitext(os.path.split(hooks_path)[1])
+            if hooks_file_ext == '.py':
+                _log.info("Importing hooks implementation from %s...", hooks_path)
+                try:
+                    # import module that defines hooks, and collect all functions of which name ends with '_hook'
+                    imported_hooks = imp.load_source(hooks_filename, hooks_path)
+                    for attr in dir(imported_hooks):
+                        if attr.endswith(HOOK_SUFF):
+                            hook = getattr(imported_hooks, attr)
+                            if callable(hook):
+                                hooks.update({attr: hook})
+                            else:
+                                _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
+                    _log.info("Found hooks: %s", sorted(hooks.keys()))
+                except ImportError as err:
+                    raise EasyBuildError("Failed to import hooks implementation from %s: %s", hooks_path, err)
+            else:
+                raise EasyBuildError("Provided path for hooks implementation should be path to a Python file (*.py)")
+        else:
+            _log.info("No location for hooks implementation provided, no hooks defined")
 
-    verify_hooks(hooks)
+        verify_hooks(hooks)
+
+        # cache loaded hooks, so we don't need to load them from file again
+        _cached_hooks[hooks_path] = hooks
 
     return hooks
 
@@ -157,7 +170,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
     return res
 
 
-def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None):
+def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
     """
     Run hook with specified label.
 
@@ -166,6 +179,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None)
     :param pre_step_hook: indicates whether hook to run is a pre-step hook
     :param post_step_hook: indicates whether hook to run is a post-step hook
     :param args: arguments to pass to hook function
+    :param msg: custom message that is printed when hook is called
     """
     hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
     if hook:
@@ -177,6 +191,9 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None)
         elif post_step_hook:
             label = 'post-' + label
 
-        print_msg("Running %s hook..." % label)
+        if msg is None:
+            msg = "Running %s hook..." % label
+        print_msg(msg)
+
         _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
         hook(*args)

test/framework/hooks.py

@@ -32,8 +32,9 @@
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 from unittest import TextTestRunner
 
+import easybuild.tools.hooks
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import write_file
+from easybuild.tools.filetools import remove_file, write_file
 from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks
 
 
@@ -48,6 +49,9 @@ def setUp(self):
             'def start_hook():',
             '    print("this is triggered at the very beginning")',
             '',
+            'def parse_hook(ec):',
+            '   print("Parse hook with argument %s" % ec)',
+            '',
             'def foo():',
             '    print("running foo helper method")',
             '',
@@ -67,10 +71,29 @@ def test_load_hooks(self):
 
         hooks = load_hooks(self.test_hooks_pymod)
 
-        self.assertEqual(len(hooks), 3)
-        self.assertEqual(sorted(hooks.keys()), ['post_configure_hook', 'pre_install_hook', 'start_hook'])
+        self.assertEqual(len(hooks), 4)
+        self.assertEqual(sorted(hooks.keys()), ['parse_hook', 'post_configure_hook', 'pre_install_hook', 'start_hook'])
         self.assertTrue(all(callable(h) for h in hooks.values()))
 
+        # test caching of hooks
+        remove_file(self.test_hooks_pymod)
+        cached_hooks = load_hooks(self.test_hooks_pymod)
+        self.assertTrue(cached_hooks is hooks)
+
+        # hooks file can be empty
+        empty_hooks_path = os.path.join(self.test_prefix, 'empty_hooks.py')
+        write_file(empty_hooks_path, '')
+        empty_hooks = load_hooks(empty_hooks_path)
+        self.assertEqual(empty_hooks, {})
+
+        # loading another hooks file doesn't affect cached hooks
+        prev_hooks = load_hooks(self.test_hooks_pymod)
+        self.assertTrue(prev_hooks is hooks)
+
+        # clearing cached hooks results in error because hooks file is not found
+        easybuild.tools.hooks._cached_hooks = {}
+        self.assertErrorRegex(EasyBuildError, "Specified path .* does not exist.*", load_hooks, self.test_hooks_pymod)
+
     def test_find_hook(self):
         """Test for find_hook function."""
 
@@ -104,6 +127,7 @@ def test_run_hook(self):
         self.mock_stdout(True)
         self.mock_stderr(True)
         run_hook('start', hooks)
+        run_hook('parse', hooks, args=['<EasyConfig instance>'], msg="Running parse hook for example.eb...")
         run_hook('configure', hooks, pre_step_hook=True, args=[None])
         run_hook('configure', hooks, post_step_hook=True, args=[None])
         run_hook('build', hooks, pre_step_hook=True, args=[None])
@@ -118,6 +142,8 @@ def test_run_hook(self):
         expected_stdout = '\n'.join([
             "== Running start hook...",
             "this is triggered at the very beginning",
+            "== Running parse hook for example.eb...",
+            "Parse hook with argument <EasyConfig instance>",
             "== Running post-configure hook...",
             "this is run after configure step",
             "running foo helper method",