Merge branch 'develop' into 5.0.x

RELEASE_NOTES

@@ -4,6 +4,28 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.2 (12 June 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - improve behavior when using extension which has 'nosource' enabled (#4506)
+  - enhance 'get_software_libdir' to return 'lib' or 'lib64' if only one of them contains library files (#4513)
+  - implement versions checks to avoid mixing major versions across the EasyBuild components (#4520, #4553)
+  - add support for easyconfig parameter 'module_only' (#4537)
+- various bug fixes, including:
+  - fix typo in patch_step logging (#4505)
+  - consider both 'easybuild-framework*.tar.gz' and 'easybuild_framework*.tar.gz' in CI workflows (#4507)
+  - don't delete existing environment module files when using '--dump-env-script' with '--force' or '--rebuild' (#4512)
+  - fix resolved (template) values in case of failure  (#4532)
+  - also consider '$CRAY_PE_LIBSCI_PREFIX_DIR' to determine installation prefix for cray-libsci (#4551)
+  - symlink downloaded repo at specified commit when using '--from-commit' so easyconfigs for dependencies are found (#4552)
+- other changes:
+  - code cleanup in 'easyblock.py' (#4519)
+  - stop running unit tests on Python 3.5 (#4530)
+
+
 v4.9.1 (5 April 2024)
 ---------------------
 

easybuild/framework/easyblock.py

@@ -1060,7 +1060,7 @@ def make_builddir(self):
             self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) "
                           "and 'keeppreviousinstall' because we're building in the installation directory.")
             # force cleanup before installation
-            if build_option('module_only'):
+            if build_option('module_only') or self.cfg['module_only']:
                 self.log.debug("Disabling cleanupoldbuild because we run as module-only")
                 self.cfg['cleanupoldbuild'] = False
             else:
@@ -1131,7 +1131,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
             if self.cfg['keeppreviousinstall']:
                 self.log.info("Keeping old directory %s (hopefully you know what you are doing)", dir_name)
                 return
-            elif build_option('module_only'):
+            elif build_option('module_only') or self.cfg['module_only']:
                 self.log.info("Not touching existing directory %s in module-only mode...", dir_name)
             elif clean:
                 remove_dir(dir_name)
@@ -2111,7 +2111,7 @@ def guess_start_dir(self):
         start_dir = ''
         # do not use the specified 'start_dir' when running as --module-only as
         # the directory will not exist (extract_step is skipped)
-        if self.start_dir and not build_option('module_only'):
+        if self.start_dir and not build_option('module_only') and not self.cfg['module_only']:
             start_dir = self.start_dir
 
         if not os.path.isabs(start_dir):
@@ -3813,7 +3813,7 @@ def make_module_step(self, fake=False):
                 try:
                     self.make_devel_module()
                 except EasyBuildError as error:
-                    if build_option('module_only'):
+                    if build_option('module_only') or self.cfg['module_only']:
                         self.log.info("Using --module-only so can recover from error: %s", error)
                     else:
                         raise error
@@ -3921,7 +3921,7 @@ def skip_step(self, step, skippable):
         """Dedice whether or not to skip the specified step."""
         skip = False
         force = build_option('force')
-        module_only = build_option('module_only')
+        module_only = build_option('module_only') or self.cfg['module_only']
         sanity_check_only = build_option('sanity_check_only')
         skip_extensions = build_option('skip_extensions')
         skip_test_step = build_option('skip_test_step')

easybuild/framework/easyconfig/default.py

@@ -109,6 +109,7 @@
     'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD],
     'installopts': ['', 'Extra options for installation', BUILD],
     'maxparallel': [None, 'Max degree of parallelism', BUILD],
+    'module_only': [False, 'Only generate module file', BUILD],
     'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of '
                         'cores, active cpuset and restrictions in ulimit)'), BUILD],
     'patches': [[], "List of patches to apply", BUILD],

easybuild/main.py

@@ -82,6 +82,8 @@
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.systemtools import check_easybuild_deps
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
+from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_EASYBLOCKS_VERSION
+from easybuild.tools.version import different_major_versions
 
 
 _log = None
@@ -614,6 +616,15 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
     (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate,
      from_pr_list, tweaked_ecs_paths) = cfg_settings
 
+    # compare running Framework and EasyBlocks versions
+    if EASYBLOCKS_VERSION == UNKNOWN_EASYBLOCKS_VERSION:
+        # most likely reason is running framework unit tests with no easyblocks installation
+        # so log a warning, to avoid test related issues
+        _log.warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework")
+    elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION):
+        raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION,
+                                                                                                  EASYBLOCKS_VERSION))
+
     # load hook implementations (if any)
     hooks = load_hooks(options.hooks)
 

easybuild/toolchains/linalg/libsci.py

@@ -65,13 +65,20 @@ def _get_software_root(self, name, required=True):
         """Get install prefix for specified software name; special treatment for Cray modules."""
         if name == 'cray-libsci':
             # Cray-provided LibSci module
-            env_var = 'CRAY_LIBSCI_PREFIX_DIR'
-            root = os.getenv(env_var, None)
+            root = None
+            # consider both $CRAY_LIBSCI_PREFIX_DIR and $CRAY_PE_LIBSCI_PREFIX_DIR,
+            # cfr. https://github.com/easybuilders/easybuild-framework/issues/4536
+            env_vars = ('CRAY_LIBSCI_PREFIX_DIR', 'CRAY_PE_LIBSCI_PREFIX_DIR')
+            for env_var in env_vars:
+                root = os.getenv(env_var, None)
+                if root is not None:
+                    self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root)
+                    break
+
             if root is None:
                 if required:
-                    raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var)
-            else:
-                self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root)
+                    env_vars_str = ', '.join('$' + e for e in env_vars)
+                    raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_vars_str)
         else:
             root = super(LibSci, self)._get_software_root(name, required=required)
 

easybuild/tools/github.py

@@ -61,6 +61,7 @@
 from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file
 from easybuild.tools.systemtools import UNKNOWN, get_tool_version
 from easybuild.tools.utilities import nub, only_if_module_is_available
+from easybuild.tools.version import FRAMEWORK_VERSION, different_major_versions
 
 
 _log = fancylogger.getLogger('github', fname=False)
@@ -588,9 +589,45 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
         else:
             raise EasyBuildError("Couldn't find path to patched file %s", full_path)
 
+    if github_repo == GITHUB_EASYCONFIGS_REPO:
+        ver_file = os.path.join(final_path, 'setup.py')
+    elif github_repo == GITHUB_EASYBLOCKS_REPO:
+        ver_file = os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py')
+    else:
+        raise EasyBuildError("Don't know how to determine version for repo %s", github_repo)
+
+    # take into account that the file we need to determine repo version may not be available,
+    # for example when a closed PR is used (since then we only download files patched by the PR)
+    if os.path.exists(ver_file):
+        ver = _get_version_for_repo(ver_file)
+        if different_major_versions(FRAMEWORK_VERSION, ver):
+            raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)",
+                                 FRAMEWORK_VERSION, github_account, github_repo, pr, ver)
+
     return files
 
 
+def _get_version_for_repo(filename):
+    """Extract version from filename."""
+    _log.debug("Extract version from %s" % filename)
+
+    try:
+        ver_line = ""
+        with open(filename) as f:
+            for line in f.readlines():
+                if line.startswith("VERSION "):
+                    ver_line = line
+                    break
+
+        # version can be a string or LooseVersion
+        res = re.search(r"""^VERSION = .*['"](.*)['"].?$""", ver_line)
+
+        _log.debug("PR target version is %s" % res.group(1))
+        return res.group(1)
+    except Exception:
+        raise EasyBuildError("Couldn't determine version of PR from %s" % filename)
+
+
 def fetch_easyblocks_from_pr(pr, path=None, github_user=None):
     """Fetch patched easyblocks for a particular PR."""
     return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO)
@@ -613,6 +650,13 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
     if github_repo is None:
         github_repo = GITHUB_EASYCONFIGS_REPO
 
+    if github_repo == GITHUB_EASYCONFIGS_REPO:
+        easybuild_subdir = os.path.join('easybuild', 'easyconfigs')
+    elif github_repo == GITHUB_EASYBLOCKS_REPO:
+        easybuild_subdir = os.path.join('easybuild', 'easyblocks')
+    else:
+        raise EasyBuildError("Unknown repo: %s", github_repo)
+
     if path is None:
         if github_repo == GITHUB_EASYCONFIGS_REPO:
             extra_ec_paths = build_option('extra_ec_paths')
@@ -656,6 +700,12 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
     else:
         raise EasyBuildError("Unknown repo: %s" % github_repo)
 
+    # symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path
+    mkdir(path, parents=True)
+    dirpath = os.path.join(repo_commit, easybuild_subdir)
+    for subdir in os.listdir(dirpath):
+        symlink(os.path.join(dirpath, subdir), os.path.join(path, subdir))
+
     # copy specified files to directory where they're expected to be found
     file_paths = []
     for file in files:

easybuild/tools/modules.py

@@ -37,6 +37,7 @@
 * Jens Timmerman (Ghent University)
 * David Brown (Pacific Northwest National Laboratory)
 """
+import glob
 import os
 import re
 import shlex
@@ -51,6 +52,7 @@
 from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.run import run_shell_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.utilities import get_subclasses, nub
 
 # software root/version environment variable name prefixes
@@ -1656,6 +1658,7 @@ def get_software_libdir(name, only_one=True, fs=None):
 
     Returns the library subdirectory, relative to software root.
     It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs.
+    If only_one is True and fs is None, select the one subdirectory with shared or static libraries, if possible.
 
     :param name: name of the software package
     :param only_one: indicates whether only one lib path is expected to be found
@@ -1688,6 +1691,16 @@ def get_software_libdir(name, only_one=True, fs=None):
             if len(res) == 1:
                 res = res[0]
             else:
+                if fs is None and len(res) == 2:
+                    # if both lib and lib64 were found, check if only one (exactly) has libraries;
+                    # this is needed for software with library archives in lib64 but other files/directories in lib
+                    lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]]
+                    has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res]
+                    if has_libs[0] and not has_libs[1]:
+                        return res[0]
+                    elif has_libs[1] and not has_libs[0]:
+                        return res[1]
+
                 raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s",
                                      name, root, ', '.join(res))
         return res

easybuild/tools/version.py

@@ -47,6 +47,7 @@
 # This causes problems further up the dependency chain...
 VERSION = LooseVersion('5.0.0.dev0')
 UNKNOWN = 'UNKNOWN'
+UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
 
 
 def get_git_revision():
@@ -87,7 +88,7 @@ def get_git_revision():
 try:
     from easybuild.easyblocks import VERBOSE_VERSION as EASYBLOCKS_VERSION
 except Exception:
-    EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'  # make sure it is smaller then anything
+    EASYBLOCKS_VERSION = UNKNOWN_EASYBLOCKS_VERSION  # make sure it is smaller then anything
 
 
 def this_is_easybuild():
@@ -102,3 +103,14 @@ def this_is_easybuild():
         msg = msg.encode('ascii')
 
     return msg
+
+
+def different_major_versions(v1, v2):
+    """Compare major versions"""
+    # Deal with version instances being either strings or LooseVersion
+    if isinstance(v1, str):
+        v1 = LooseVersion(v1)
+    if isinstance(v2, str):
+        v2 = LooseVersion(v2)
+
+    return v1.version[0] != v2.version[0]

test/framework/modules.py

@@ -51,6 +51,7 @@
 from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version
 from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches
 from easybuild.tools.run import run_shell_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext
 
 
 # number of modules included for testing purposes
@@ -691,10 +692,23 @@ def test_get_software_root_version_libdir(self):
             os.environ.pop('EBROOT%s' % env_var_name)
             os.environ.pop('EBVERSION%s' % env_var_name)
 
-        # check expected result of get_software_libdir with multiple lib subdirs
+        # if only 'lib' has a library archive, use it
         root = os.path.join(tmpdir, name)
         mkdir(os.path.join(root, 'lib64'))
         os.environ['EBROOT%s' % env_var_name] = root
+        write_file(os.path.join(root, 'lib', 'libfoo.a'), 'foo')
+        self.assertEqual(get_software_libdir(name), 'lib')
+
+        remove_file(os.path.join(root, 'lib', 'libfoo.a'))
+
+        # also check vice versa with *shared* library in lib64
+        shlib_ext = get_shared_lib_ext()
+        write_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext), 'foo')
+        self.assertEqual(get_software_libdir(name), 'lib64')
+
+        remove_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext))
+
+        # check expected result of get_software_libdir with multiple lib subdirs
         self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name)
         self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64'])