diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4bc5be0f7f..7e8562e7b8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1652,7 +1652,7 @@ def make_module_step(self, fake=False): self.log.info("Module file %s written" % self.module_generator.filename) - # only update after generating final module file + # only update after generating final module file if not fake: self.modules_tool.update() diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 278791512f..a26f611f4a 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -98,6 +98,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'test_report_env_filter', 'testoutput', 'umask', + 'update_modules_tool_cache', ], False: [ 'allow_modules_tool_mismatch', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 67623e8169..bdc3c8dfd0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -133,6 +133,19 @@ def hexdigest(self): return '0x%s' % (self.checksum & 0xffffffff) +def copy_file(src, dest): + """ + Copy file at specified path to new location. + @param src: source path + @param dest: destination path (full path, incl. filename) + """ + try: + mkdir(os.path.dirname(dest)) + shutil.copy2(src, dest) + except OSError, err: + _log.error("Failed to copy %s to %s: %s") + + def read_file(path, log_error=True): """Read contents of file at given path, in a robust way.""" f = None diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b1e99c92fe..a57c194f8a 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -45,15 +45,18 @@ from vsc.utils.missing import get_subclasses from vsc.utils.patterns import Singleton -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import convert_name, mkdir, read_file, path_matches, which +from easybuild.tools.filetools import convert_name, copy_file, mkdir, read_file, path_matches, which, write_file from easybuild.tools.module_naming_scheme import DEVEL_MODULE_SUFFIX from easybuild.tools.run import run_cmd from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION from vsc.utils.missing import nub +# relative to user's home directory +LMOD_USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') + # software root/version environment variable name prefixes ROOT_ENV_VAR_NAME_PREFIX = "EBROOT" VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION" @@ -139,6 +142,11 @@ class ModulesTool(object): __metaclass__ = Singleton + @staticmethod + def is_lmod(): + """Return whether this instance provides an interface to Lmod.""" + return False + def __init__(self, mod_paths=None, testing=False): """ Create a ModulesTool object @@ -468,6 +476,10 @@ def run_module(self, *args, **kwargs): # run these in terse mode for easier machine reading args.insert(*self.TERSE_OPTION) + # -D enables verbose debug logging to stderr + if self.is_lmod() and 'debug' in kwargs and kwargs['debug']: + args.insert(0, '-D') + module_path_key = None if 'mod_paths' in kwargs: module_path_key = 'mod_paths' @@ -796,15 +808,45 @@ class Lmod(ModulesTool): VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') + @staticmethod + def is_lmod(): + """Return whether this instance provides an interface to Lmod.""" + return True + def __init__(self, *args, **kwargs): """Constructor, set lmod-specific class variable values.""" + # $LMOD_QUIET needs to be set to avoid EasyBuild tripping over fiddly bits in output os.environ['LMOD_QUIET'] = '1' - # make sure Lmod ignores the spider cache ($LMOD_IGNORE_CACHE supported since Lmod 5.2) + + # make sure Lmod ignores the spider cache by defining $LMOD_IGNORE_CACHE + # this must be set early, before initializing rest of this Lmod class instance os.environ['LMOD_IGNORE_CACHE'] = '1' super(Lmod, self).__init__(*args, **kwargs) + self.lua_binpath, self.lua_ver = None, None + self.init_lua() + + def init_lua(self): + """Initialize Lua-related class variables.""" + + # determine path to lua/luac via Lmod config + lmod_config = self.run_module('--config', return_output=True) + lua_binpath_regex = re.compile(r"^Path to Lua\s*(.*)$", re.M) + res = lua_binpath_regex.search(lmod_config) + if res: + self.lua_binpath = res.group(1) + self.log.debug("Found Lua (bin)path: %s" % self.lua_binpath) + else: + self.log.error("Failed to determine Lua path from Lmod config") + + # determine Lua version + lua_bin = os.path.join(self.lua_binpath, 'lua') + lua_ver_cmd = 'print((_VERSION:gsub("Lua ","")))' + (out, _) = run_cmd("%s -e '%s'" % (lua_bin, lua_ver_cmd), simple=False) + self.lua_ver = out.strip() + def check_module_function(self, *args, **kwargs): """Check whether selected module tool matches 'module' function definition.""" if not 'regex' in kwargs: @@ -835,33 +877,75 @@ def available(self, mod_name=None): return correct_real_mods def update(self): - """Update after new modules were added.""" - if build_option('update_modules_tool_cache'): + """Update Lmod cache after new modules were added.""" + cache_dir = None + cache_timestamp = None + cache_specs = build_option('update_modules_tool_cache') + if cache_specs: + if isinstance(cache_specs, bool): + cache_specs = LMOD_USER_CACHE_DIR + cache_specs = cache_specs.split(',') + cache_dir = cache_specs[0] + if len(cache_specs) > 1: + cache_timestamp = cache_specs[1] + + if cache_dir is not None: + # type of cache to build (possible choices: moduleT, dbT, reverseMapT) + # just building moduleT is sufficient for speedy 'avail' + cache_type = 'moduleT' + + if cache_timestamp is not None: + if os.path.exists(cache_timestamp): + os.remove(cache_timestamp) + write_file(cache_timestamp, '', append=True) + + cache_fp = os.path.join(cache_dir, '%s.lua' % cache_type) + print_msg("updating Lmod cache %s..." % cache_fp, log=self.log) + + # run spider to create cache contents spider_cmd = os.path.join(os.path.dirname(self.cmd), 'spider') - cmd = [spider_cmd, '-o', 'moduleT', os.environ['MODULEPATH']] + cmd = [spider_cmd, '-o', cache_type, os.environ['MODULEPATH']] self.log.debug("Running command '%s'..." % ' '.join(cmd)) + # we can't use run_cmd, since we need to separate stdout from stderr proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ) - (stdout, stderr) = proc.communicate() + (cache_txt, stderr) = proc.communicate() if stderr: self.log.error("An error occured when running '%s': %s" % (' '.join(cmd), stderr)) if self.testing: # don't actually update local cache when testing, just return the cache contents - return stdout + return cache_txt else: + compiled_cache_fp = os.path.join(cache_dir, '%s.luac_%s' % (cache_type, self.lua_ver)) + self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) + + # back existing cache file by copying them T.old. + # Lmod also considers this filename for the cache in case T. isn't found + if os.path.exists(cache_fp): + copy_file(cache_fp, os.path.join(cache_dir, '%s.old.lua' % cache_type)) + if os.path.exists(compiled_cache_fp): + copy_file(compiled_cache_fp, os.path.join(cache_dir, '%s.old.luac_%s' % (cache_type, self.lua_ver))) + + # make update as atomic as possible: write cache file under temporary name, and then rename + new_cache_fp = '%s.new' % cache_fp + write_file(new_cache_fp, cache_txt) try: - cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') - self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) - cache_dir = os.path.dirname(cache_fp) - if not os.path.exists(cache_dir): - mkdir(cache_dir, parents=True) - cache_file = open(cache_fp, 'w') - cache_file.write(stdout) - cache_file.close() - except (IOError, OSError), err: - self.log.error("Failed to update Lmod spider cache %s: %s" % (cache_fp, err)) + os.rename(new_cache_fp, cache_fp) + except OSError, err: + self.log.error("Failed to rename Lmod cache %s to %s: %s" % (new_cache_fp, cache_fp, err)) + + # compile cache if luac is available + luac = os.path.join(self.lua_binpath, 'luac') + new_compiled_cache_fp = '%s.new' % compiled_cache_fp + if os.path.exists(luac): + run_cmd("%s -o %s.new %s" % (luac, compiled_cache_fp, cache_fp)) + try: + os.rename(new_compiled_cache_fp, compiled_cache_fp) + except OSError, err: + tup = (new_compiled_cache_fp, compiled_cache_fp, err) + self.log.error("Failed to rename Lmod cache %s to %s: %s" % tup) def prepend_module_path(self, path): # Lmod pushes a path to the front on 'module use' @@ -976,15 +1060,20 @@ def avail_modules_tools(): return class_dict -def modules_tool(mod_paths=None, testing=False): +def modules_tool(mod_paths=None, testing=False, name=None, return_class=False): """ Return interface to modules tool (environment modules (C, Tcl), or Lmod) """ + if name is None: + name = get_modules_tool() + # get_modules_tool might return none (e.g. if config was not initialized yet) - modules_tool = get_modules_tool() - if modules_tool is not None: - modules_tool_class = avail_modules_tools().get(modules_tool) - return modules_tool_class(mod_paths=mod_paths, testing=testing) + if name is not None: + modules_tool_class = avail_modules_tools().get(name) + if return_class: + return modules_tool_class + else: + return modules_tool_class(mod_paths=mod_paths, testing=testing) else: return None diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 59e6b0e5f6..696beeff62 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -58,6 +58,7 @@ from easybuild.tools.modules import avail_modules_tools from easybuild.tools.module_naming_scheme import GENERAL_CLASS from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes +from easybuild.tools.modules import LMOD_USER_CACHE_DIR, modules_tool from easybuild.tools.ordereddict import OrderedDict from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.repository.repository import avail_repositories @@ -205,7 +206,7 @@ def override_options(self): 'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed", None, 'store', None), 'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file", - None, 'store_true', False), + None, 'store_or_None', True, {'metavar': 'DIR[,FILE]'}), }) self.log.debug("override_options: descr %s opts %s" % (descr, opts))