Merge branch 'develop' into 5.0.x

.github/workflows/end2end.yml

@@ -17,6 +17,7 @@ jobs:
       fail-fast: false
     container:
       image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

.github/workflows/unit_tests.yml

@@ -187,7 +187,7 @@ jobs:
             # run test suite
             python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
             # try and make sure output of running tests is clean (no printed messages/warnings)
-            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved|algorithms.TripleDES"
             # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
             PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
             test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)

easybuild/framework/easyblock.py

@@ -666,14 +666,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         src_path = ext_src['src']
                         src_fn = os.path.basename(src_path)
 
+                        src_checksums = {}
                         for checksum_type in [CHECKSUM_TYPE_SHA256]:
                             src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+                            src_checksums[checksum_type] = src_checksum
                             self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
 
                         # verify checksum (if provided)
                         self.log.debug('Verifying checksums for extension source...')
                         fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
-                        if verify_checksum(src_path, fn_checksum):
+                        if verify_checksum(src_path, fn_checksum, src_checksums):
                             self.log.info('Checksum for extension source %s verified', src_fn)
                         elif build_option('ignore_checksums'):
                             print_warning("Ignoring failing checksum verification for %s" % src_fn)
@@ -692,10 +694,13 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         ext_src.update({'patches': ext_patches})
 
                         if verify_checksums:
+                            computed_checksums = {}
                             for patch in ext_patches:
                                 patch = patch['path']
+                                computed_checksums[patch] = {}
                                 for checksum_type in [CHECKSUM_TYPE_SHA256]:
                                     checksum = compute_checksum(patch, checksum_type=checksum_type)
+                                    computed_checksums[patch][checksum_type] = checksum
                                     self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
 
                             # verify checksum (if provided)
@@ -705,7 +710,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                                 patch_fn = os.path.basename(patch)
 
                                 checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1)
-                                if verify_checksum(patch, checksum):
+                                if verify_checksum(patch, checksum, computed_checksums[patch]):
                                     self.log.info('Checksum for extension patch %s verified', patch_fn)
                                 elif build_option('ignore_checksums'):
                                     print_warning("Ignoring failing checksum verification for %s" % patch_fn)
@@ -744,7 +749,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         """
         srcpaths = source_paths()
 
-        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
+        # We don't account for the checksums file in the progress bar
+        if filename != 'checksum.json':
+            update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
 
         if alt_location is None:
             location = self.name

easybuild/framework/easyconfig/templates.py

@@ -93,9 +93,11 @@
      "as specify by the --sysroot configuration option"),
     ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
     ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
-     "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
+     "--cuda-compute-capabilities configuration option or via cuda_cc easyconfig parameter"),
     ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
     ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
+    ('cuda_cc_space_sep_no_period',
+     "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."),
     ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
     ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"),
     ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"),
@@ -449,17 +451,18 @@ def template_constant_dict(config, ignore=None, toolchain=None):
 
     # step 6. CUDA compute capabilities
     #         Use the commandline / easybuild config option if given, else use the value from the EC (as a default)
-    cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
-    if cuda_compute_capabilities:
-        template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities)
-        template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
-        template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
-        template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
-        int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities]
+    cuda_cc = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
+    if cuda_cc:
+        template_values['cuda_compute_capabilities'] = ','.join(cuda_cc)
+        template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc)
+        template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc)
+        template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc)
+        template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc)
+        int_values = [cc.replace('.', '') for cc in cuda_cc]
         template_values['cuda_int_comma_sep'] = ','.join(int_values)
         template_values['cuda_int_space_sep'] = ' '.join(int_values)
         template_values['cuda_int_semicolon_sep'] = ';'.join(int_values)
-        sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
+        sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_cc]
         template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
         template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
 

easybuild/framework/easyconfig/tools.py

@@ -54,7 +54,7 @@
 from easybuild.framework.easyconfig.easyconfig import process_easyconfig
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
 from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
 from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files
@@ -218,10 +218,12 @@ def mk_node_name(spec):
             if dep in spec['ec'].build_dependencies:
                 dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs)
 
-    _dep_graph_dump(dgr, filename)
-
-    if not build_option('silent'):
-        print("Wrote dependency graph for %d easyconfigs to %s" % (len(specs), filename))
+    what = "dependency graph for %d easyconfigs to %s" % (len(specs), filename)
+    silent = build_option('silent')
+    if _dep_graph_dump(dgr, filename):
+        print_msg("Wrote " + what, silent=silent)
+    else:
+        print_error("Failed writing " + what, silent=silent)
 
 
 @only_if_module_is_available('pygraph.readwrite.dot', pkgname='python-graph-dot')
@@ -231,18 +233,24 @@ def _dep_graph_dump(dgr, filename):
     dottxt = dot.write(dgr)
     if os.path.splitext(filename)[-1] == '.dot':
         # create .dot file
-        write_file(filename, dottxt)
+        try:
+            write_file(filename, dottxt)
+        except EasyBuildError as e:
+            print(str(e))
+            return False
+        else:
+            return True
     else:
-        _dep_graph_gv(dottxt, filename)
+        return _dep_graph_gv(dottxt, filename)
 
 
 @only_if_module_is_available('gv', pkgname='graphviz-python')
 def _dep_graph_gv(dottxt, filename):
     """Render dependency graph to file using graphviz."""
     # try and render graph in specified file format
     gvv = gv.readstring(dottxt)
-    gv.layout(gvv, 'dot')
-    gv.render(gvv, os.path.splitext(filename)[-1], filename)
+    if gv.layout(gvv, 'dot') is not False:
+        return gv.render(gvv, os.path.splitext(filename)[-1], filename)
 
 
 def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):

easybuild/toolchains/compiler/gcc.py

@@ -78,6 +78,15 @@ class Gcc(Compiler):
         COMPILER_UNIQUE_OPTION_MAP['strict'] = no_recip_alternative
         COMPILER_UNIQUE_OPTION_MAP['precise'] = no_recip_alternative
 
+    # gcc on RISC-V does not support -mno-recip, -mieee-fp, -mfno-math-errno...
+    # https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Options.html
+    # there are no good alternatives, so stick to the default flags
+    if systemtools.get_cpu_family() == systemtools.RISCV:
+        COMPILER_UNIQUE_OPTION_MAP['strict'] = []
+        COMPILER_UNIQUE_OPTION_MAP['precise'] = []
+        COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno']
+        COMPILER_UNIQUE_OPTION_MAP['verloose'] = ['fno-math-errno']
+
     # used when 'optarch' toolchain option is enabled (and --optarch is not specified)
     COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
         (systemtools.AARCH32, systemtools.ARM): 'mcpu=native',  # implies -march=native and -mtune=native

easybuild/toolchains/compiler/intel_compilers.py

@@ -48,7 +48,8 @@ class IntelCompilers(IntelIccIfort):
         'oneapi': (None, "Use oneAPI compilers icx/icpx/ifx instead of classic compilers"),
         'oneapi_c_cxx': (None, "Use oneAPI C/C++ compilers icx/icpx instead of classic Intel C/C++ compilers "
                                "(auto-enabled for Intel compilers version 2022.2.0, or newer)"),
-        'oneapi_fortran': (False, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler"),
+        'oneapi_fortran': (None, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler "
+                                 "(auto-enabled for Intel compilers version 2024.0.0, or newer)"),
     })
 
     def _set_compiler_vars(self):
@@ -75,6 +76,9 @@ def set_variables(self):
         # auto-enable use of oneAPI C/C++ compilers for sufficiently recent versions of Intel compilers
         comp_ver = self.get_software_version(self.COMPILER_MODULE_NAME)[0]
         if LooseVersion(comp_ver) >= LooseVersion('2022.2.0'):
+            if LooseVersion(comp_ver) >= LooseVersion('2024.0.0'):
+                if self.options.get('oneapi_fortran', None) is None:
+                    self.options['oneapi_fortran'] = True
             if self.options.get('oneapi_c_cxx', None) is None:
                 self.options['oneapi_c_cxx'] = True
 

easybuild/toolchains/gmpflf.py

@@ -0,0 +1,45 @@
+##
+# Copyright 2013-2024 Ghent University
+#
+# This file is triple-licensed under GPLv2 (see below), MIT, and
+# BSD three-clause licenses.
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH, FlexiBLAS, LAPACK, ScaLAPACK and FFTW).
+
+Authors:
+
+* Richard Topouchian (University of Bergen)
+"""
+from easybuild.toolchains.gmpich import Gmpich
+from easybuild.toolchains.gfbf import Gfbf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Gmpflf(Gmpich, FlexiBLAS,  ScaLAPACK, Fftw):
+    """Compiler toolchain with GCC, MPICH, FlexiBLAS, ScaLAPACK and FFTW."""
+    NAME = 'gmpflf'
+    SUBTOOLCHAIN = [Gmpich.NAME, Gfbf.NAME]

easybuild/toolchains/linalg/intelmkl.py

@@ -139,13 +139,16 @@ def _set_blas_variables(self):
             self.variables.nappend_el('CFLAGS', 'DMKL_ILP64')
 
         # exact paths/linking statements depend on imkl version
+        root = self.get_software_root(self.BLAS_MODULE_NAME)[0]
         found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0]
         ver = LooseVersion(found_version)
         if ver < LooseVersion('10.3'):
             self.BLAS_LIB_DIR = ['lib/em64t']
             self.BLAS_INCLUDE_DIR = ['include']
         else:
             if ver >= LooseVersion('2021'):
+                if os.path.islink(os.path.join(root, 'mkl', 'latest')):
+                    found_version = os.readlink(os.path.join(root, 'mkl', 'latest'))
                 basedir = os.path.join('mkl', found_version)
             else:
                 basedir = 'mkl'

easybuild/tools/filetools.py

@@ -1250,12 +1250,15 @@ def calc_block_checksum(path, algorithm):
     return algorithm.hexdigest()
 
 
-def verify_checksum(path, checksums):
+def verify_checksum(path, checksums, computed_checksums=None):
     """
     Verify checksum of specified file.
 
     :param path: path of file to verify checksum of
     :param checksums: checksum values (and type, optionally, default is sha256), e.g., 'af314', ('sha', '5ec1b')
+    :param computed_checksums: Optional dictionary of (current) checksum(s) for this file
+                               indexed by the checksum type (e.g. 'sha256').
+                               Each existing entry will be used, missing ones will be computed.
     """
 
     filename = os.path.basename(path)
@@ -1311,8 +1314,14 @@ def verify_checksum(path, checksums):
                                  "2-tuple (type, value), or tuple of alternative checksum specs.",
                                  checksum)
 
-        actual_checksum = compute_checksum(path, typ)
-        _log.debug("Computed %s checksum for %s: %s (correct checksum: %s)" % (typ, path, actual_checksum, checksum))
+        if computed_checksums is not None and typ in computed_checksums:
+            actual_checksum = computed_checksums[typ]
+            computed_str = 'Precomputed'
+        else:
+            actual_checksum = compute_checksum(path, typ)
+            computed_str = 'Computed'
+        _log.debug("%s %s checksum for %s: %s (correct checksum: %s)" %
+                   (computed_str, typ, path, actual_checksum, checksum))
 
         if actual_checksum != checksum:
             return False

easybuild/tools/options.py

@@ -1585,7 +1585,7 @@ def check_included_multiple(included_easyblocks_from, source):
                 check_included_multiple(included_from_commit, "commit %s" % easyblock_commit)
 
             for easyblock in included_from_commit:
-                print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log)
+                print_msg("easyblock %s included from commit %s" % (easyblock, easyblock_commit), log=log)
 
             include_easyblocks(options.tmpdir, easyblocks_from_commit)
 

test/framework/easyconfig.py

@@ -4624,7 +4624,7 @@ def test_cuda_compute_capabilities(self):
             prebuildopts = '%(cuda_cc_semicolon_sep)s'
             buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" '
                          'semi="%(cuda_int_semicolon_sep)s"')
-            preinstallopts = '%(cuda_cc_space_sep)s'
+            preinstallopts = 'period="%(cuda_cc_space_sep)s" noperiod="%(cuda_cc_space_sep_no_period)s"'
             installopts = '%(cuda_compute_capabilities)s'
         """)
         self.prep()
@@ -4637,7 +4637,7 @@ def test_cuda_compute_capabilities(self):
         self.assertEqual(ec['buildopts'], 'comma="51,70,71" '
                                           'space="51 70 71" '
                                           'semi="51;70;71"')
-        self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1')
+        self.assertEqual(ec['preinstallopts'], 'period="5.1 7.0 7.1" noperiod="51 70 71"')
         self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
 
         # build options overwrite it
@@ -4650,7 +4650,7 @@ def test_cuda_compute_capabilities(self):
                                           'space="42 63" '
                                           'semi="42;63"')
         self.assertEqual(ec['prebuildopts'], '4.2;6.3')
-        self.assertEqual(ec['preinstallopts'], '4.2 6.3')
+        self.assertEqual(ec['preinstallopts'], 'period="4.2 6.3" noperiod="42 63"')
         self.assertEqual(ec['installopts'], '4.2,6.3')
 
     def test_det_copy_ec_specs(self):