diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index a43ce28535..94b66db903 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
+v4.9.0 (30 December 2023)
+-------------------------
+
+feature release
+
+- various enhancements, including:
+ - allow tweaking of easyconfigs from different toolchains (#3669)
+ - add support for `--list-software --output-format=json` (#4152)
+ - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403)
+- various bug fixes, including:
+ - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361)
+ - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369)
+ - adapt module function check for Environment Modules v4+ (#4371)
+ - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375)
+ - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377)
+ - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379)
+ - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385)
+ - eliminate duplicate multideps from generated module files (#4386)
+ - resolve templated values in extension names in `_make_extension_list` (#4392)
+ - use source toolchain version when passing only `--try-toolchain` (#4395)
+ - fix writing spider cache for Lmod >= 8.7.12 (#4402)
+ - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405)
+ - fix `LooseVersion` when running with Python 2.7 (#4408)
+ - use more recent easyblocks PR in `test_github_merge_pr` (#4414)
+- other changes:
+ - extend test that checks build environment to recent `foss/2023a` toolchain (#4391)
+
+
v4.8.2 (29 October 2023)
------------------------
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 536242e690..20605c185a 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -4651,7 +4651,7 @@ def inject_checksums(ecs, checksum_type):
def make_list_lines(values, indent_level):
"""Make lines for list of values."""
def to_str(s):
- if isinstance(s, string_type):
+ if isinstance(s, str):
return "'%s'" % s
else:
return str(s)
diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py
index c6e1403ab5..a008ae3959 100644
--- a/easybuild/tools/docs.py
+++ b/easybuild/tools/docs.py
@@ -38,6 +38,7 @@
"""
import copy
import inspect
+import json
import os
from collections import OrderedDict
from easybuild.tools import LooseVersion
@@ -73,6 +74,7 @@
DETAILED = 'detailed'
SIMPLE = 'simple'
+FORMAT_JSON = 'json'
FORMAT_MD = 'md'
FORMAT_RST = 'rst'
FORMAT_TXT = 'txt'
@@ -116,6 +118,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT):
return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants])
+def avail_cfgfile_constants_json(go_cfg_constants):
+ """Generate documentation on constants for configuration files in json format"""
+ raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json")
+
+
def avail_cfgfile_constants_txt(go_cfg_constants):
"""Generate documentation on constants for configuration files in txt format"""
doc = [
@@ -185,6 +192,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT):
return generate_doc('avail_easyconfig_constants_%s' % output_format, [])
+def avail_easyconfig_constants_json():
+ """Generate easyconfig constant documentation in json format"""
+ raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json")
+
+
def avail_easyconfig_constants_txt():
"""Generate easyconfig constant documentation in txt format"""
doc = ["Constants that can be used in easyconfigs"]
@@ -243,6 +255,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT):
return generate_doc('avail_easyconfig_licenses_%s' % output_format, [])
+def avail_easyconfig_licenses_json():
+ """Generate easyconfig license documentation in json format"""
+ raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json")
+
+
def avail_easyconfig_licenses_txt():
"""Generate easyconfig license documentation in txt format"""
doc = ["License constants that can be used in easyconfigs"]
@@ -355,6 +372,13 @@ def avail_easyconfig_params_rst(title, grouped_params):
return '\n'.join(doc)
+def avail_easyconfig_params_json():
+ """
+ Compose overview of available easyconfig parameters, in json format.
+ """
+ raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json")
+
+
def avail_easyconfig_params_txt(title, grouped_params):
"""
Compose overview of available easyconfig parameters, in plain text format.
@@ -427,6 +451,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT):
return generate_doc('avail_easyconfig_templates_%s' % output_format, [])
+def avail_easyconfig_templates_json():
+ """ Returns template documentation in json text format """
+ raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates")
+
+
def avail_easyconfig_templates_txt():
""" Returns template documentation in plain text format """
# This has to reflect the methods/steps used in easyconfig _generate_template_values
@@ -641,6 +670,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings
def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT):
+ if output_format == FORMAT_JSON:
+ raise NotImplementedError("JSON output format not supported for list_easyblocks")
format_strings = {
FORMAT_MD: {
'det_root_templ': "- **%s** (%s%s)",
@@ -1025,6 +1056,38 @@ def list_software_txt(software, detailed=False):
return '\n'.join(lines)
+def list_software_json(software, detailed=False):
+ """
+ Return overview of supported software in json
+
+ :param software: software information (strucuted like list_software does)
+ :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+ :return: multi-line string presenting requested info
+ """
+ lines = ['[']
+ for key in sorted(software, key=lambda x: x.lower()):
+ for entry in software[key]:
+ if detailed:
+ # deep copy here to avoid modifying the original dict
+ entry = copy.deepcopy(entry)
+ entry['description'] = ' '.join(entry['description'].split('\n')).strip()
+ else:
+ entry = {}
+ entry['name'] = key
+
+ lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",")
+ if not detailed:
+ break
+
+ # remove trailing comma on last line
+ if len(lines) > 1:
+ lines[-1] = lines[-1].rstrip(',')
+
+ lines.append(']')
+
+ return '\n'.join(lines)
+
+
def list_toolchains(output_format=FORMAT_TXT):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
@@ -1173,6 +1236,11 @@ def list_toolchains_txt(tcs):
return '\n'.join(doc)
+def list_toolchains_json(tcs):
+ """ Returns overview of all toolchains in json format """
+ raise NotImplementedError("JSON output not implemented yet for --list-toolchains")
+
+
def avail_toolchain_opts(name, output_format=FORMAT_TXT):
"""Show list of known options for given toolchain."""
tc_class, _ = search_toolchain(name)
@@ -1226,6 +1294,11 @@ def avail_toolchain_opts_rst(name, tc_dict):
return '\n'.join(doc)
+def avail_toolchain_opts_json(name, tc_dict):
+ """ Returns overview of toolchain options in jsonformat """
+ raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts")
+
+
def avail_toolchain_opts_txt(name, tc_dict):
""" Returns overview of toolchain options in txt format """
doc = ["Available options for %s toolchain:" % name]
@@ -1252,6 +1325,13 @@ def get_easyblock_classes(package_name):
return easyblocks
+def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None):
+ """
+ Compose overview of all easyblocks in the given package in json format
+ """
+ raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview")
+
+
def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None):
"""
Compose overview of all easyblocks in the given package in MarkDown format
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index aa0b59c4ae..253dce094a 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -79,7 +79,7 @@
from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
from easybuild.tools.config import BuildOptions, ConfigurationVariables
from easybuild.tools.configobj import ConfigObj, ConfigObjError
-from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT
+from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT
from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates
from easybuild.tools.docs import list_easyblocks, list_toolchains
@@ -466,7 +466,8 @@ def override_options(self):
'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True),
'optarch': ("Set architecture optimization, overriding native architecture optimizations",
None, 'store', None),
- 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]),
+ 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT,
+ [FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT]),
'output-style': ("Control output style; auto implies using Rich if available to produce rich output, "
"with fallback to basic colored output",
'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),
diff --git a/test/framework/docs.py b/test/framework/docs.py
index 84d862bd3c..70280892e4 100644
--- a/test/framework/docs.py
+++ b/test/framework/docs.py
@@ -405,6 +405,104 @@
``1.4``|``GCC/4.6.3``, ``system``
``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+LIST_SOFTWARE_SIMPLE_MD = """# List of supported software
+
+EasyBuild supports 2 different software packages (incl. toolchains, bundles):
+
+[g](#g)
+
+
+## G
+
+* GCC
+* gzip"""
+
+LIST_SOFTWARE_DETAILED_MD = """# List of supported software
+
+EasyBuild supports 2 different software packages (incl. toolchains, bundles):
+
+[g](#g)
+
+
+## G
+
+
+[GCC](#gcc) - [gzip](#gzip)
+
+
+### GCC
+
+%(gcc_descr)s
+
+*homepage*:
+
+version |toolchain
+---------|----------
+``4.6.3``|``system``
+
+### gzip
+
+%(gzip_descr)s
+
+*homepage*:
+
+version|toolchain
+-------|-------------------------------
+``1.4``|``GCC/4.6.3``, ``system``
+``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+
+LIST_SOFTWARE_SIMPLE_JSON = """[
+{
+ "name": "GCC"
+},
+{
+ "name": "gzip"
+}
+]"""
+
+LIST_SOFTWARE_DETAILED_JSON = """[
+{
+ "description": "%(gcc_descr)s",
+ "homepage": "http://gcc.gnu.org/",
+ "name": "GCC",
+ "toolchain": "system",
+ "version": "4.6.3",
+ "versionsuffix": ""
+},
+{
+ "description": "%(gzip_descr)s",
+ "homepage": "http://www.gzip.org/",
+ "name": "gzip",
+ "toolchain": "GCC/4.6.3",
+ "version": "1.4",
+ "versionsuffix": ""
+},
+{
+ "description": "%(gzip_descr)s",
+ "homepage": "http://www.gzip.org/",
+ "name": "gzip",
+ "toolchain": "system",
+ "version": "1.4",
+ "versionsuffix": ""
+},
+{
+ "description": "%(gzip_descr)s",
+ "homepage": "http://www.gzip.org/",
+ "name": "gzip",
+ "toolchain": "foss/2018a",
+ "version": "1.5",
+ "versionsuffix": ""
+},
+{
+ "description": "%(gzip_descr)s",
+ "homepage": "http://www.gzip.org/",
+ "name": "gzip",
+ "toolchain": "intel/2018a",
+ "version": "1.5",
+ "versionsuffix": ""
+}
+]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR}
+
class DocsTest(EnhancedTestCase):
@@ -541,6 +639,9 @@ def test_license_docs(self):
regex = re.compile(r"^``GPLv3``\s*|The GNU General Public License", re.M)
self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs))
+ # expect NotImplementedError for JSON output
+ self.assertRaises(NotImplementedError, avail_easyconfig_licenses, output_format='json')
+
def test_list_easyblocks(self):
"""
Tests for list_easyblocks function
@@ -569,6 +670,9 @@ def test_list_easyblocks(self):
txt = list_easyblocks(list_easyblocks='detailed', output_format='md')
self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_MD % {'topdir': topdir_easyblocks})
+ # expect NotImplementedError for JSON output
+ self.assertRaises(NotImplementedError, list_easyblocks, output_format='json')
+
def test_list_software(self):
"""Test list_software* functions."""
build_options = {
@@ -587,6 +691,9 @@ def test_list_software(self):
self.assertEqual(list_software(output_format='md'), LIST_SOFTWARE_SIMPLE_MD)
self.assertEqual(list_software(output_format='md', detailed=True), LIST_SOFTWARE_DETAILED_MD)
+ self.assertEqual(list_software(output_format='json'), LIST_SOFTWARE_SIMPLE_JSON)
+ self.assertEqual(list_software(output_format='json', detailed=True), LIST_SOFTWARE_DETAILED_JSON)
+
# GCC/4.6.3 is installed, no gzip module installed
txt = list_software(output_format='txt', detailed=True, only_installed=True)
self.assertTrue(re.search(r'^\* GCC', txt, re.M))
@@ -690,6 +797,10 @@ def test_list_toolchains(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
+ # expect NotImplementedError for json output format
+ with self.assertRaises(NotImplementedError):
+ list_toolchains(output_format='json')
+
def test_avail_cfgfile_constants(self):
"""
Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file.
@@ -734,6 +845,10 @@ def test_avail_cfgfile_constants(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
+ # expect NotImplementedError for json output format
+ with self.assertRaises(NotImplementedError):
+ avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='json')
+
def test_avail_easyconfig_constants(self):
"""
Test avail_easyconfig_constants to generate overview of constants that can be used in easyconfig files.
@@ -777,6 +892,10 @@ def test_avail_easyconfig_constants(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
+ # expect NotImplementedError for json output format
+ with self.assertRaises(NotImplementedError):
+ avail_easyconfig_constants(output_format='json')
+
def test_avail_easyconfig_templates(self):
"""
Test avail_easyconfig_templates to generate overview of templates that can be used in easyconfig files.
@@ -827,6 +946,10 @@ def test_avail_easyconfig_templates(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
+ # expect NotImplementedError for json output format
+ with self.assertRaises(NotImplementedError):
+ avail_easyconfig_templates(output_format='json')
+
def test_avail_toolchain_opts(self):
"""
Test avail_toolchain_opts to generate overview of supported toolchain options.
@@ -911,6 +1034,12 @@ def test_avail_toolchain_opts(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst))
+ # expect NotImplementedError for json output format
+ with self.assertRaises(NotImplementedError):
+ avail_toolchain_opts('foss', output_format='json')
+ with self.assertRaises(NotImplementedError):
+ avail_toolchain_opts('intel', output_format='json')
+
def test_mk_table(self):
"""
Tests for mk_*_table functions.