Skip to content

Commit

Permalink
Merge pull request #5 from boegel/cug20
Browse files Browse the repository at this point in the history 
fixes, cleanup, enhancements and tests for probing external modules for missing metadata
  • Loading branch information
@victorusu
victorusu authored Apr 9, 2020
2 parents b80c8d6 + 227a9fd commit 1fbea39
Show file tree
Hide file tree
Showing 5 changed files with 292 additions and 120 deletions.
187 changes: 104 additions & 83 deletions easybuild/framework/easyconfig/easyconfig.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1166,37 +1166,54 @@ def _validate(self, attr, values): # private method
if self[attr] and self[attr] not in values:
raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values)

def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=None):
def probe_external_module_metadata(self, mod_name, existing_metadata=None):
"""
helper function for handle_external_module_metadata
handles metadata for external module dependencies when there is not entry in the
metadata file
It should look for the pair of variables definitions in the available modules
1. CRAY_XXXX_PREFIX and CRAY_XXXX_VERSION
2. CRAY_XXXX_DIR and CRAY_XXXX_VERSION
2. CRAY_XXXX_ROOT and CRAY_XXXX_VERSION
5. XXXX_PREFIX and XXXX_VERSION
4. XXXX_DIR and XXXX_VERSION
5. XXXX_ROOT and XXXX_VERSION
3. XXXX_HOME and XXXX_VERSION
If neither of the pairs is found, then an empty dictionary is returned
Helper function for handle_external_module_metadata.
Tries to determine metadata for external module when there is not entry in the metadata file,
by looking at the variables defined by the module file.
This is mainly intended for modules provided in the Cray Programming Environment,
but it could also be useful in other contexts.
The following pairs of variables are considered (in order, first hit wins),
where 'XXX' is the software name in capitals:
1. $CRAY_XXX_PREFIX and $CRAY_XXX_VERSION
1. $CRAY_XXX_PREFIX_DIR and $CRAY_XXX_VERSION
2. $CRAY_XXX_DIR and $CRAY_XXX_VERSION
2. $CRAY_XXX_ROOT and $CRAY_XXX_VERSION
5. $XXX_PREFIX and $XXX_VERSION
4. $XXX_DIR and $XXX_VERSION
5. $XXX_ROOT and $XXX_VERSION
3. $XXX_HOME and $XXX_VERSION
If none of the pairs is found, then an empty dictionary is returned.
:param mod_name: name of the external module
:param metadata: already available metadata for this external module (if any)
"""
if dependency is None:
dependency = dict()
res = {}

short_ext_modname = dep_name.split('/')[0]
if not 'name' in dependency:
dependency['name'] = [short_ext_modname]
if existing_metadata is None:
existing_metadata = {}

if short_ext_modname.startswith('cray-'):
short_ext_modname = short_ext_modname.split('cray-')[1]
soft_name = existing_metadata.get('name')
if soft_name:
# software name is a list of names in metadata, just grab first one
soft_name = soft_name[0]
else:
# if the software name is not known yet, use the first part of the module name as software name,
# but strip off the leading 'cray-' part first (examples: cray-netcdf/4.6.1.3, cray-fftw/3.3.8.2)
soft_name = mod_name.split('/')[0]

cray_prefix = 'cray-'
if soft_name.startswith(cray_prefix):
soft_name = soft_name[len(cray_prefix):]

short_ext_modname.replace('-', '_')
short_ext_modname_upper = convert_name(short_ext_modname, upper=True)
# determine software name to use in names of environment variables (upper case, '-' becomes '_')
soft_name_in_mod_name = convert_name(soft_name.replace('-', '_'), upper=True)

allowed_pairs = [
var_name_pairs = [
('CRAY_%s_PREFIX', 'CRAY_%s_VERSION'),
('CRAY_%s_PREFIX_DIR', 'CRAY_%s_VERSION'),
('CRAY_%s_DIR', 'CRAY_%s_VERSION'),
Expand All @@ -1207,71 +1224,75 @@ def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=No
('%s_HOME', '%s_VERSION'),
]

for prefix, version in allowed_pairs:
prefix = prefix % short_ext_modname_upper
version = version % short_ext_modname_upper

dep_prefix = self.modules_tool.get_variable_from_modulefile(dep_name, prefix)
dep_version = self.modules_tool.get_variable_from_modulefile(dep_name, version)

# only update missing values with both keys are found
if dep_prefix and dep_version:
# version should hold the value, not the key
if 'version' not in dependency:
dependency['version'] = [dep_version]
self.log.info('setting external module %s version to be %s' % (dep_name, dep_version))
# prefix should hold the key, not the value
if 'prefix' not in dependency:
dependency['prefix'] = prefix
self.log.info('setting external module %s prefix to be %s' % (dep_name, dep_prefix))
for prefix_var_name, version_var_name in var_name_pairs:
prefix_var_name = prefix_var_name % soft_name_in_mod_name
version_var_name = version_var_name % soft_name_in_mod_name

prefix = self.modules_tool.get_setenv_value_from_modulefile(mod_name, prefix_var_name)
version = self.modules_tool.get_setenv_value_from_modulefile(mod_name, version_var_name)

# we only have a hit when values for *both* variables are found
if prefix and version:

if 'name' not in existing_metadata:
res['name'] = [soft_name]

# if a version is already set in the available metadata, we retain it
if 'version' not in existing_metadata:
res['version'] = [version]
self.log.info('setting external module %s version to be %s', mod_name, version)

# if a prefix is already set in the available metadata, we retain it
if 'prefix' not in existing_metadata:
res['prefix'] = prefix
self.log.info('setting external module %s prefix to be %s', mod_name, prefix_var_name)
break

return dependency
return res

def handle_external_module_metadata(self, dep_name):
def handle_external_module_metadata(self, mod_name):
"""
helper function for _parse_dependency
handles metadata for external module dependencies
Helper function for _parse_dependency; collects metadata for external module dependencies.
:param mod_name: name of external module to collect metadata for
"""
dependency = {}
dep_name_no_version = dep_name.split('/')[0]
metadata_fields = ['name', 'version', 'prefix']
external_metadata = {}

if dep_name in self.external_modules_metadata:
external_metadata = self.external_modules_metadata[dep_name]
if not all(d in external_metadata for d in metadata_fields):
external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name,
dependency=external_metadata)
if external_metadata:
self.log.info("Updated dependency info with metadata from available modules for external module %s: %s",
dep_name, external_metadata)
dependency['external_module_metadata'] = external_metadata
partial_mod_name = mod_name.split('/')[0]

# check whether existing metadata for external modules already has metadata for this module;
# first using full module name (as it is provided), for example 'cray-netcdf/4.6.1.3',
# then with partial module name, for example 'cray-netcdf'
metadata = self.external_modules_metadata.get(mod_name, {})
self.log.info("Available metadata for external module %s: %s", mod_name, metadata)

partial_mod_name_metadata = self.external_modules_metadata.get(partial_mod_name, {})
self.log.info("Available metadata for external module using partial module name %s: %s",
partial_mod_name, partial_mod_name_metadata)

for key in partial_mod_name_metadata:
if key not in metadata:
metadata[key] = partial_mod_name_metadata[key]

self.log.info("Combined available metadata for external module %s: %s", mod_name, metadata)

# if not all metadata is available (name/version/prefix), probe external module to collect more metadata;
# first with full module name, and then with partial module name if first probe didn't return anything;
# note: result of probe_external_module_metadata only contains metadata for keys that were not set yet
if not all(key in metadata for key in ['name', 'prefix', 'version']):
self.log.info("Not all metadata found yet for external module %s, probing module...", mod_name)
probed_metadata = self.probe_external_module_metadata(mod_name, existing_metadata=metadata)
if probed_metadata:
self.log.info("Extra metadata found by probing external module %s: %s", mod_name, probed_metadata)
metadata.update(probed_metadata)
else:
self.log.info("No metadata available for external module %s.", dep_name)
elif dep_name_no_version in self.external_modules_metadata:
external_metadata = self.external_modules_metadata[dep_name_no_version]
if not all(d in external_metadata for d in metadata_fields):
external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name_no_version,
dependency=external_metadata)
if external_metadata:
self.log.info("Updated dependency info with metadata from available modules for external module "
"%s: %s", dep_name, external_metadata)
dependency['external_module_metadata'] = external_metadata
else:
self.log.info("No metadata available for external module %s.", dep_name)
else:
self.log.info("No metadata available for external module %s. Attempting to read from available modules",
dep_name)
external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name)
if external_metadata:
dependency['external_module_metadata'] = external_metadata
self.log.info("Updated dependency info with metadata from available modules for external module %s: %s",
dep_name, external_metadata)
else:
self.log.info("No metadata available for external module %s.", dep_name)
self.log.info("No extra metadata found by probing %s, trying with partial module name...", mod_name)
probed_metadata = self.probe_external_module_metadata(partial_mod_name, existing_metadata=metadata)
self.log.info("Extra metadata for external module %s found by probing partial module name %s: %s",
mod_name, partial_mod_name, probed_metadata)
metadata.update(probed_metadata)

return dependency
self.log.info("Obtained metadata after module probing: %s", metadata)

return {'external_module_metadata': metadata}

def handle_multi_deps(self):
"""
Expand Down
81 changes: 48 additions & 33 deletions easybuild/tools/modules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -649,33 +649,28 @@ def show(self, mod_name):

return ans

def get_variable_from_modulefile(self, mod_name, var_name):
"""
Get info from the module file for the specified module.
:param mod_name: module name
:param var_name: name of the variable value to be extracted
"""
pass

def get_value_from_modulefile(self, mod_name, regex):
def get_value_from_modulefile(self, mod_name, regex, strict=True):
"""
Get info from the module file for the specified module.
:param mod_name: module name
:param regex: (compiled) regular expression, with one group
"""
value = None

if self.exist([mod_name], skip_avail=True)[0]:
modinfo = self.show(mod_name)
res = regex.search(modinfo)
if res:
return res.group(1)
else:
value = res.group(1)
elif strict:
raise EasyBuildError("Failed to determine value from 'show' (pattern: '%s') in %s",
regex.pattern, modinfo)
else:
elif strict:
raise EasyBuildError("Can't get value from a non-existing module %s", mod_name)

return value

def modulefile_path(self, mod_name, strip_ext=False):
"""
Get the path of the module file for the specified module
Expand Down Expand Up @@ -1095,6 +1090,15 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps,
self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path))
return path

def get_setenv_value_from_modulefile(self, mod_name, var_name):
"""
Get value for specific 'setenv' statement from module file for the specified module.
:param mod_name: module name
:param var_name: name of the variable being set for which value should be returned
"""
raise NotImplementedError

def update(self):
"""Update after new modules were added."""
raise NotImplementedError
Expand Down Expand Up @@ -1135,21 +1139,26 @@ def update(self):
"""Update after new modules were added."""
pass

def get_variable_from_modulefile(self, mod_name, var_name):
def get_setenv_value_from_modulefile(self, mod_name, var_name):
"""
Get info from the module file for the specified module.
Get value for specific 'setenv' statement from module file for the specified module.
:param mod_name: module name
:param var_name: name of the variable value to be extracted
:param var_name: name of the variable being set for which value should be returned
"""
try:
# Tcl syntax
regex = re.compile(r'^setenv\s+%s\s+(?P<value>\S*)' % var_name, re.M)
ans = self.get_value_from_modulefile(mod_name, regex)
except Exception:
return None
# Tcl-based module tools produce "module show" output with setenv statements like:
# "setenv GCC_PATH /opt/gcc/8.3.0"
# - line starts with 'setenv'
# - whitespace (spaces & tabs) around variable name
# - no quotes or parentheses around value (which can contain spaces!)
regex = re.compile(r'^setenv\s+%s\s+(?P<value>.+)' % var_name, re.M)
value = self.get_value_from_modulefile(mod_name, regex, strict=False)

if value:
value = value.strip()

return value

return ans

class EnvironmentModulesTcl(EnvironmentModulesC):
"""Interface to (Tcl) environment modules (modulecmd.tcl)."""
Expand Down Expand Up @@ -1414,21 +1423,27 @@ def exist(self, mod_names, skip_avail=False, maybe_partial=True):
return super(Lmod, self).exist(mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*(\.lua)?:\s*$',
skip_avail=skip_avail, maybe_partial=maybe_partial)

def get_variable_from_modulefile(self, mod_name, var_name):
def get_setenv_value_from_modulefile(self, mod_name, var_name):
"""
Get info from the module file for the specified module.
Get value for specific 'setenv' statement from module file for the specified module.
:param mod_name: module name
:param var_name: name of the variable value to be extracted
:param var_name: name of the variable being set for which value should be returned
"""
try:
# Lua syntax
regex = re.compile(r'^setenv\(\"%s\",\s*\"(?P<value>\S*)\"\)' % var_name, re.M)
ans = self.get_value_from_modulefile(mod_name, regex)
except Exception:
return None
# Lmod produces "module show" output with setenv statements like:
# setenv("EBROOTBZIP2","/tmp/software/bzip2/1.0.6")
# - line starts with setenv(
# - both variable name and value are enclosed in double quotes, separated by comma
# - value can contain spaces!
# - line ends with )
regex = re.compile(r'^setenv\("%s"\s*,\s*"(?P<value>.+)"\)' % var_name, re.M)
value = self.get_value_from_modulefile(mod_name, regex, strict=False)

if value:
value = value.strip()

return value

return ans

def get_software_root_env_var_name(name):
"""Return name of environment variable for software root."""
Expand Down
Loading

0 comments on commit 1fbea39

Please sign in to comment.