Merge branch '5.0.x' into system_tcopts

.github/workflows/end2end.yml

@@ -17,6 +17,7 @@ jobs:
       fail-fast: false
     container:
       image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

.github/workflows/unit_tests.yml

@@ -101,12 +101,12 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
         if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
           fi
           echo "GitHub token installed!"
         else
@@ -187,7 +187,17 @@ jobs:
             # run test suite
             python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
             # try and make sure output of running tests is clean (no printed messages/warnings)
-            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test|CryptographyDeprecationWarning: TripleDES has been moved|algorithms.TripleDES"
+            IGNORE_PATTERNS="no GitHub token available"
+            IGNORE_PATTERNS+="|skipping SvnRepository test"
+            IGNORE_PATTERNS+="|requires Lmod as modules tool"
+            IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+            IGNORE_PATTERNS+="|from cryptography.* import "
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
+            IGNORE_PATTERNS+="|Blowfish"
+            IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+            IGNORE_PATTERNS+="|algorithms.TripleDES"
             # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
             PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
             test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)

easybuild/framework/easyblock.py

@@ -666,14 +666,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         src_path = ext_src['src']
                         src_fn = os.path.basename(src_path)
 
+                        src_checksums = {}
                         for checksum_type in [CHECKSUM_TYPE_SHA256]:
                             src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+                            src_checksums[checksum_type] = src_checksum
                             self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
 
                         # verify checksum (if provided)
                         self.log.debug('Verifying checksums for extension source...')
                         fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
-                        if verify_checksum(src_path, fn_checksum):
+                        if verify_checksum(src_path, fn_checksum, src_checksums):
                             self.log.info('Checksum for extension source %s verified', src_fn)
                         elif build_option('ignore_checksums'):
                             print_warning("Ignoring failing checksum verification for %s" % src_fn)
@@ -692,10 +694,13 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         ext_src.update({'patches': ext_patches})
 
                         if verify_checksums:
+                            computed_checksums = {}
                             for patch in ext_patches:
                                 patch = patch['path']
+                                computed_checksums[patch] = {}
                                 for checksum_type in [CHECKSUM_TYPE_SHA256]:
                                     checksum = compute_checksum(patch, checksum_type=checksum_type)
+                                    computed_checksums[patch][checksum_type] = checksum
                                     self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
 
                             # verify checksum (if provided)
@@ -705,7 +710,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                                 patch_fn = os.path.basename(patch)
 
                                 checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1)
-                                if verify_checksum(patch, checksum):
+                                if verify_checksum(patch, checksum, computed_checksums[patch]):
                                     self.log.info('Checksum for extension patch %s verified', patch_fn)
                                 elif build_option('ignore_checksums'):
                                     print_warning("Ignoring failing checksum verification for %s" % patch_fn)
@@ -744,7 +749,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         """
         srcpaths = source_paths()
 
-        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
+        # We don't account for the checksums file in the progress bar
+        if filename != 'checksum.json':
+            update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
 
         if alt_location is None:
             location = self.name

easybuild/framework/easyconfig/default.py

@@ -108,7 +108,7 @@
                        BUILD],
     'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD],
     'installopts': ['', 'Extra options for installation', BUILD],
-    'maxparallel': [None, 'Max degree of parallelism', BUILD],
+    'maxparallel': [16, 'Max degree of parallelism', BUILD],
     'module_only': [False, 'Only generate module file', BUILD],
     'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of '
                         'cores, active cpuset and restrictions in ulimit)'), BUILD],

easybuild/framework/easyconfig/easyconfig.py

@@ -46,8 +46,8 @@
 import functools
 import os
 import re
-from contextlib import contextmanager
 from collections import OrderedDict
+from contextlib import contextmanager
 
 import easybuild.tools.filetools as filetools
 from easybuild.base import fancylogger

easybuild/framework/easyconfig/templates.py

@@ -90,18 +90,21 @@
 TEMPLATE_NAMES_DYNAMIC = [
     ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
     ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
-     "as specify by the --sysroot configuration option"),
+     "as specified by the --sysroot configuration option"),
     ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
     ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
-     "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
+     "--cuda-compute-capabilities configuration option or via cuda_cc easyconfig parameter"),
     ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
     ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
+    ('cuda_cc_space_sep_no_period',
+     "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."),
     ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
     ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"),
     ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"),
     ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"),
     ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
     ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
+    ('software_commit', "Git commit id to use for the software as specified by --software-commit command line option"),
 ]
 
 # constant templates that can be used in easyconfigs
@@ -130,7 +133,9 @@
     ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
      'download.savannah.gnu.org source url'),
     ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',
-     'gnu.org source url'),
+     'gnu.org source url (ftp mirror)'),
+    ('GNU_FTP_SOURCE', 'https://ftp.gnu.org/gnu/%(namelower)s',
+     'gnu.org source url (main ftp)'),
     ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files',
      'googlecode.com source url'),
     ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/',
@@ -214,6 +219,7 @@
     'GITHUB_RELEASE_URL': 'GITHUB_RELEASE',
     'GITHUB_RELEASE_URL_LOWER': 'GITHUB_LOWER_RELEASE',
     'GNU_SAVANNAH_URL': 'GNU_SAVANNAH_SOURCE',
+    'GNU_FTP_URL': 'GNU_FTP_SOURCE',
     'GNU_URL': 'GNU_SOURCE',
     'GOOGLECODE_URL': 'GOOGLECODE_SOURCE',
     'LAUNCHPAD_URL': 'LAUNCHPAD_SOURCE',
@@ -292,6 +298,9 @@ def template_constant_dict(config, ignore=None, toolchain=None):
     # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback
     template_values['sysroot'] = build_option('sysroot') or ''
 
+    # set 'software_commit' template based on 'software_commit' configuration option, default to None
+    template_values['software_commit'] = build_option('software_commit') or ''
+
     # step 1: add TEMPLATE_NAMES_EASYCONFIG
     for name in TEMPLATE_NAMES_EASYCONFIG:
         if name in ignore:
@@ -449,17 +458,18 @@ def template_constant_dict(config, ignore=None, toolchain=None):
 
     # step 6. CUDA compute capabilities
     #         Use the commandline / easybuild config option if given, else use the value from the EC (as a default)
-    cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
-    if cuda_compute_capabilities:
-        template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities)
-        template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
-        template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
-        template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
-        int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities]
+    cuda_cc = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
+    if cuda_cc:
+        template_values['cuda_compute_capabilities'] = ','.join(cuda_cc)
+        template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc)
+        template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc)
+        template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc)
+        template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc)
+        int_values = [cc.replace('.', '') for cc in cuda_cc]
         template_values['cuda_int_comma_sep'] = ','.join(int_values)
         template_values['cuda_int_space_sep'] = ' '.join(int_values)
         template_values['cuda_int_semicolon_sep'] = ';'.join(int_values)
-        sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
+        sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_cc]
         template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
         template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
 

easybuild/framework/easyconfig/tools.py

@@ -54,7 +54,7 @@
 from easybuild.framework.easyconfig.easyconfig import process_easyconfig
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
 from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
 from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files
@@ -218,10 +218,12 @@ def mk_node_name(spec):
             if dep in spec['ec'].build_dependencies:
                 dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs)
 
-    _dep_graph_dump(dgr, filename)
-
-    if not build_option('silent'):
-        print("Wrote dependency graph for %d easyconfigs to %s" % (len(specs), filename))
+    what = "dependency graph for %d easyconfigs to %s" % (len(specs), filename)
+    silent = build_option('silent')
+    if _dep_graph_dump(dgr, filename):
+        print_msg("Wrote " + what, silent=silent)
+    else:
+        print_error("Failed writing " + what, silent=silent)
 
 
 @only_if_module_is_available('pygraph.readwrite.dot', pkgname='python-graph-dot')
@@ -231,18 +233,24 @@ def _dep_graph_dump(dgr, filename):
     dottxt = dot.write(dgr)
     if os.path.splitext(filename)[-1] == '.dot':
         # create .dot file
-        write_file(filename, dottxt)
+        try:
+            write_file(filename, dottxt)
+        except EasyBuildError as e:
+            print(str(e))
+            return False
+        else:
+            return True
     else:
-        _dep_graph_gv(dottxt, filename)
+        return _dep_graph_gv(dottxt, filename)
 
 
 @only_if_module_is_available('gv', pkgname='graphviz-python')
 def _dep_graph_gv(dottxt, filename):
     """Render dependency graph to file using graphviz."""
     # try and render graph in specified file format
     gvv = gv.readstring(dottxt)
-    gv.layout(gvv, 'dot')
-    gv.render(gvv, os.path.splitext(filename)[-1], filename)
+    if gv.layout(gvv, 'dot') is not False:
+        return gv.render(gvv, os.path.splitext(filename)[-1], filename)
 
 
 def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):

easybuild/toolchains/compiler/gcc.py

@@ -78,6 +78,15 @@ class Gcc(Compiler):
         COMPILER_UNIQUE_OPTION_MAP['strict'] = no_recip_alternative
         COMPILER_UNIQUE_OPTION_MAP['precise'] = no_recip_alternative
 
+    # gcc on RISC-V does not support -mno-recip, -mieee-fp, -mfno-math-errno...
+    # https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Options.html
+    # there are no good alternatives, so stick to the default flags
+    if systemtools.get_cpu_family() == systemtools.RISCV:
+        COMPILER_UNIQUE_OPTION_MAP['strict'] = []
+        COMPILER_UNIQUE_OPTION_MAP['precise'] = []
+        COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno']
+        COMPILER_UNIQUE_OPTION_MAP['verloose'] = ['fno-math-errno']
+
     # used when 'optarch' toolchain option is enabled (and --optarch is not specified)
     COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
         (systemtools.AARCH32, systemtools.ARM): 'mcpu=native',  # implies -march=native and -mtune=native

easybuild/toolchains/compiler/intel_compilers.py

@@ -48,7 +48,8 @@ class IntelCompilers(IntelIccIfort):
         'oneapi': (None, "Use oneAPI compilers icx/icpx/ifx instead of classic compilers"),
         'oneapi_c_cxx': (None, "Use oneAPI C/C++ compilers icx/icpx instead of classic Intel C/C++ compilers "
                                "(auto-enabled for Intel compilers version 2022.2.0, or newer)"),
-        'oneapi_fortran': (False, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler"),
+        'oneapi_fortran': (None, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler "
+                                 "(auto-enabled for Intel compilers version 2024.0.0, or newer)"),
     })
 
     def _set_compiler_vars(self):
@@ -75,6 +76,9 @@ def set_variables(self):
         # auto-enable use of oneAPI C/C++ compilers for sufficiently recent versions of Intel compilers
         comp_ver = self.get_software_version(self.COMPILER_MODULE_NAME)[0]
         if LooseVersion(comp_ver) >= LooseVersion('2022.2.0'):
+            if LooseVersion(comp_ver) >= LooseVersion('2024.0.0'):
+                if self.options.get('oneapi_fortran', None) is None:
+                    self.options['oneapi_fortran'] = True
             if self.options.get('oneapi_c_cxx', None) is None:
                 self.options['oneapi_c_cxx'] = True
 

easybuild/toolchains/gmpflf.py

@@ -0,0 +1,45 @@
+##
+# Copyright 2013-2024 Ghent University
+#
+# This file is triple-licensed under GPLv2 (see below), MIT, and
+# BSD three-clause licenses.
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH, FlexiBLAS, LAPACK, ScaLAPACK and FFTW).
+
+Authors:
+
+* Richard Topouchian (University of Bergen)
+"""
+from easybuild.toolchains.gmpich import Gmpich
+from easybuild.toolchains.gfbf import Gfbf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Gmpflf(Gmpich, FlexiBLAS,  ScaLAPACK, Fftw):
+    """Compiler toolchain with GCC, MPICH, FlexiBLAS, ScaLAPACK and FFTW."""
+    NAME = 'gmpflf'
+    SUBTOOLCHAIN = [Gmpich.NAME, Gfbf.NAME]

easybuild/toolchains/linalg/intelmkl.py

@@ -139,13 +139,16 @@ def _set_blas_variables(self):
             self.variables.nappend_el('CFLAGS', 'DMKL_ILP64')
 
         # exact paths/linking statements depend on imkl version
+        root = self.get_software_root(self.BLAS_MODULE_NAME)[0]
         found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0]
         ver = LooseVersion(found_version)
         if ver < LooseVersion('10.3'):
             self.BLAS_LIB_DIR = ['lib/em64t']
             self.BLAS_INCLUDE_DIR = ['include']
         else:
             if ver >= LooseVersion('2021'):
+                if os.path.islink(os.path.join(root, 'mkl', 'latest')):
+                    found_version = os.readlink(os.path.join(root, 'mkl', 'latest'))
                 basedir = os.path.join('mkl', found_version)
             else:
                 basedir = 'mkl'

easybuild/tools/config.py

@@ -260,6 +260,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'rpath_override_dirs',
         'required_linked_shared_libs',
         'skip',
+        'software_commit',
         'stop',
         'subdir_user_modules',
         'sysroot',