add support for --update-lmod-caches (disabled by default) and --compile-lmod-caches (WIP)

easybuild/framework/easyblock.py

@@ -1652,7 +1652,7 @@ def make_module_step(self, fake=False):
 
         self.log.info("Module file %s written" % self.module_generator.filename)
 
-         # only update after generating final module file
+        # only update after generating final module file
         if not fake:
             self.modules_tool.update()
 

easybuild/tools/config.py

@@ -98,6 +98,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'test_report_env_filter',
         'testoutput',
         'umask',
+        'update_modules_tool_cache',
     ],
     False: [
         'allow_modules_tool_mismatch',

easybuild/tools/filetools.py

@@ -133,6 +133,19 @@ def hexdigest(self):
         return '0x%s' % (self.checksum & 0xffffffff)
 
 
+def copy_file(src, dest):
+    """
+    Copy file at specified path to new location.
+    @param src: source path
+    @param dest: destination path (full path, incl. filename)
+    """
+    try:
+        mkdir(os.path.dirname(dest))
+        shutil.copy2(src, dest)
+    except OSError, err:
+        _log.error("Failed to copy %s to %s: %s")
+
+
 def read_file(path, log_error=True):
     """Read contents of file at given path, in a robust way."""
     f = None

easybuild/tools/modules.py

@@ -45,15 +45,18 @@
 from vsc.utils.missing import get_subclasses
 from vsc.utils.patterns import Singleton
 
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option, get_modules_tool, install_path
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import convert_name, mkdir, read_file, path_matches, which
+from easybuild.tools.filetools import convert_name, copy_file, mkdir, read_file, path_matches, which, write_file
 from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX
 from easybuild.tools.run import run_cmd
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION
 from vsc.utils.missing import nub
 
+# relative to user's home directory
+LMOD_USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')
+
 # software root/version environment variable name prefixes
 ROOT_ENV_VAR_NAME_PREFIX = "EBROOT"
 VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION"
@@ -139,6 +142,11 @@ class ModulesTool(object):
 
     __metaclass__ = Singleton
 
+    @staticmethod
+    def is_lmod():
+        """Return whether this instance provides an interface to Lmod."""
+        return False
+
     def __init__(self, mod_paths=None, testing=False):
         """
         Create a ModulesTool object
@@ -468,6 +476,10 @@ def run_module(self, *args, **kwargs):
             # run these in terse mode for easier machine reading
             args.insert(*self.TERSE_OPTION)
 
+        # -D enables verbose debug logging to stderr
+        if self.is_lmod() and 'debug' in kwargs and kwargs['debug']:
+            args.insert(0, '-D')
+
         module_path_key = None
         if 'mod_paths' in kwargs:
             module_path_key = 'mod_paths'
@@ -796,15 +808,45 @@ class Lmod(ModulesTool):
     VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P<version>\d\S*)\s"
     USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')
 
+    @staticmethod
+    def is_lmod():
+        """Return whether this instance provides an interface to Lmod."""
+        return True
+
     def __init__(self, *args, **kwargs):
         """Constructor, set lmod-specific class variable values."""
+
         # $LMOD_QUIET needs to be set to avoid EasyBuild tripping over fiddly bits in output
         os.environ['LMOD_QUIET'] = '1'
-        # make sure Lmod ignores the spider cache ($LMOD_IGNORE_CACHE supported since Lmod 5.2)
+
+        # make sure Lmod ignores the spider cache by defining $LMOD_IGNORE_CACHE
+        # this must be set early, before initializing rest of this Lmod class instance
         os.environ['LMOD_IGNORE_CACHE'] = '1'
 
         super(Lmod, self).__init__(*args, **kwargs)
 
+        self.lua_binpath, self.lua_ver = None, None
+        self.init_lua()
+
+    def init_lua(self):
+        """Initialize Lua-related class variables."""
+
+        # determine path to lua/luac via Lmod config
+        lmod_config = self.run_module('--config', return_output=True)
+        lua_binpath_regex = re.compile(r"^Path to Lua\s*(.*)$", re.M)
+        res = lua_binpath_regex.search(lmod_config)
+        if res:
+            self.lua_binpath = res.group(1)
+            self.log.debug("Found Lua (bin)path: %s" % self.lua_binpath)
+        else:
+            self.log.error("Failed to determine Lua path from Lmod config")
+
+        # determine Lua version
+        lua_bin = os.path.join(self.lua_binpath, 'lua')
+        lua_ver_cmd = 'print((_VERSION:gsub("Lua ","")))'
+        (out, _) = run_cmd("%s -e '%s'" % (lua_bin, lua_ver_cmd), simple=False)
+        self.lua_ver = out.strip()
+
     def check_module_function(self, *args, **kwargs):
         """Check whether selected module tool matches 'module' function definition."""
         if not 'regex' in kwargs:
@@ -835,33 +877,75 @@ def available(self, mod_name=None):
         return correct_real_mods
 
     def update(self):
-        """Update after new modules were added."""
-        if build_option('update_modules_tool_cache'):
+        """Update Lmod cache after new modules were added."""
+        cache_dir = None
+        cache_timestamp = None
+        cache_specs = build_option('update_modules_tool_cache')
+        if cache_specs:
+            if isinstance(cache_specs, bool):
+                cache_specs = LMOD_USER_CACHE_DIR
+            cache_specs = cache_specs.split(',')
+            cache_dir = cache_specs[0]
+            if len(cache_specs) > 1:
+                cache_timestamp = cache_specs[1]
+
+        if cache_dir is not None:
+            # type of cache to build (possible choices: moduleT, dbT, reverseMapT)
+            # just building moduleT is sufficient for speedy 'avail'
+            cache_type = 'moduleT'
+
+            if cache_timestamp is not None:
+                if os.path.exists(cache_timestamp):
+                    os.remove(cache_timestamp)
+                write_file(cache_timestamp, '', append=True)
+
+            cache_fp = os.path.join(cache_dir, '%s.lua' % cache_type)
+            print_msg("updating Lmod cache %s..." % cache_fp, log=self.log)
+
+            # run spider to create cache contents
             spider_cmd = os.path.join(os.path.dirname(self.cmd), 'spider')
-            cmd = [spider_cmd, '-o', 'moduleT', os.environ['MODULEPATH']]
+            cmd = [spider_cmd, '-o', cache_type, os.environ['MODULEPATH']]
             self.log.debug("Running command '%s'..." % ' '.join(cmd))
 
+            # we can't use run_cmd, since we need to separate stdout from stderr
             proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ)
-            (stdout, stderr) = proc.communicate()
+            (cache_txt, stderr) = proc.communicate()
 
             if stderr:
                 self.log.error("An error occured when running '%s': %s" % (' '.join(cmd), stderr))
 
             if self.testing:
                 # don't actually update local cache when testing, just return the cache contents
-                return stdout
+                return cache_txt
             else:
+                compiled_cache_fp = os.path.join(cache_dir, '%s.luac_%s' % (cache_type, self.lua_ver))
+                self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd)))
+
+                # back existing cache file by copying them <cache>T.old.<ext>
+                # Lmod also considers this filename for the cache in case <cache>T.<ext> isn't found
+                if os.path.exists(cache_fp):
+                    copy_file(cache_fp, os.path.join(cache_dir, '%s.old.lua' % cache_type))
+                if os.path.exists(compiled_cache_fp):
+                    copy_file(compiled_cache_fp, os.path.join(cache_dir, '%s.old.luac_%s' % (cache_type, self.lua_ver)))
+
+                # make update as atomic as possible: write cache file under temporary name, and then rename
+                new_cache_fp = '%s.new' % cache_fp
+                write_file(new_cache_fp, cache_txt)
                 try:
-                    cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua')
-                    self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd)))
-                    cache_dir = os.path.dirname(cache_fp)
-                    if not os.path.exists(cache_dir):
-                        mkdir(cache_dir, parents=True)
-                    cache_file = open(cache_fp, 'w')
-                    cache_file.write(stdout)
-                    cache_file.close()
-                except (IOError, OSError), err:
-                    self.log.error("Failed to update Lmod spider cache %s: %s" % (cache_fp, err))
+                    os.rename(new_cache_fp, cache_fp)
+                except OSError, err:
+                    self.log.error("Failed to rename Lmod cache %s to %s: %s" % (new_cache_fp, cache_fp, err))
+
+                # compile cache if luac is available
+                luac = os.path.join(self.lua_binpath, 'luac')
+                new_compiled_cache_fp = '%s.new' % compiled_cache_fp
+                if os.path.exists(luac):
+                    run_cmd("%s -o %s.new %s" % (luac, compiled_cache_fp, cache_fp))
+                    try:
+                        os.rename(new_compiled_cache_fp, compiled_cache_fp)
+                    except OSError, err:
+                        tup = (new_compiled_cache_fp, compiled_cache_fp, err)
+                        self.log.error("Failed to rename Lmod cache %s to %s: %s" % tup)
 
     def prepend_module_path(self, path):
         # Lmod pushes a path to the front on 'module use'
@@ -976,15 +1060,20 @@ def avail_modules_tools():
     return class_dict
 
 
-def modules_tool(mod_paths=None, testing=False):
+def modules_tool(mod_paths=None, testing=False, name=None, return_class=False):
     """
     Return interface to modules tool (environment modules (C, Tcl), or Lmod)
     """
+    if name is None:
+        name = get_modules_tool()
+
     # get_modules_tool might return none (e.g. if config was not initialized yet)
-    modules_tool = get_modules_tool()
-    if modules_tool is not None:
-        modules_tool_class = avail_modules_tools().get(modules_tool)
-        return modules_tool_class(mod_paths=mod_paths, testing=testing)
+    if name is not None:
+        modules_tool_class = avail_modules_tools().get(name)
+        if return_class:
+            return modules_tool_class
+        else:
+            return modules_tool_class(mod_paths=mod_paths, testing=testing)
     else:
         return None
 

easybuild/tools/options.py

@@ -58,6 +58,7 @@
 from easybuild.tools.modules import avail_modules_tools
 from easybuild.tools.module_naming_scheme import GENERAL_CLASS
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes
+from easybuild.tools.modules import LMOD_USER_CACHE_DIR, modules_tool
 from easybuild.tools.ordereddict import OrderedDict
 from easybuild.tools.toolchain.utilities import search_toolchain
 from easybuild.tools.repository.repository import avail_repositories
@@ -205,7 +206,7 @@ def override_options(self):
             'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed",
                       None, 'store', None),
             'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file",
-                                          None, 'store_true', False),
+                                          None, 'store_or_None', True, {'metavar': 'DIR[,FILE]'}),
         })
 
         self.log.debug("override_options: descr %s opts %s" % (descr, opts))