diff --git a/RELEASE_NOTES b/RELEASE_NOTES index a43ce28535..94b66db903 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,34 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.0 (30 December 2023) +------------------------- + +feature release + +- various enhancements, including: + - allow tweaking of easyconfigs from different toolchains (#3669) + - add support for `--list-software --output-format=json` (#4152) + - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403) +- various bug fixes, including: + - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361) + - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369) + - adapt module function check for Environment Modules v4+ (#4371) + - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375) + - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) + - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379) + - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385) + - eliminate duplicate multideps from generated module files (#4386) + - resolve templated values in extension names in `_make_extension_list` (#4392) + - use source toolchain version when passing only `--try-toolchain` (#4395) + - fix writing spider cache for Lmod >= 8.7.12 (#4402) + - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405) + - fix `LooseVersion` when running with Python 2.7 (#4408) + - use more recent easyblocks PR in `test_github_merge_pr` (#4414) +- other changes: + - extend test that checks build environment to recent `foss/2023a` toolchain (#4391) + + v4.8.2 (29 October 2023) ------------------------ diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 536242e690..20605c185a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4651,7 +4651,7 @@ def inject_checksums(ecs, checksum_type): def make_list_lines(values, indent_level): """Make lines for list of values.""" def to_str(s): - if isinstance(s, string_type): + if isinstance(s, str): return "'%s'" % s else: return str(s) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index c6e1403ab5..a008ae3959 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -38,6 +38,7 @@ """ import copy import inspect +import json import os from collections import OrderedDict from easybuild.tools import LooseVersion @@ -73,6 +74,7 @@ DETAILED = 'detailed' SIMPLE = 'simple' +FORMAT_JSON = 'json' FORMAT_MD = 'md' FORMAT_RST = 'rst' FORMAT_TXT = 'txt' @@ -116,6 +118,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT): return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants]) +def avail_cfgfile_constants_json(go_cfg_constants): + """Generate documentation on constants for configuration files in json format""" + raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json") + + def avail_cfgfile_constants_txt(go_cfg_constants): """Generate documentation on constants for configuration files in txt format""" doc = [ @@ -185,6 +192,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_constants_%s' % output_format, []) +def avail_easyconfig_constants_json(): + """Generate easyconfig constant documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json") + + def avail_easyconfig_constants_txt(): """Generate easyconfig constant documentation in txt format""" doc = ["Constants that can be used in easyconfigs"] @@ -243,6 +255,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_licenses_%s' % output_format, []) +def avail_easyconfig_licenses_json(): + """Generate easyconfig license documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json") + + def avail_easyconfig_licenses_txt(): """Generate easyconfig license documentation in txt format""" doc = ["License constants that can be used in easyconfigs"] @@ -355,6 +372,13 @@ def avail_easyconfig_params_rst(title, grouped_params): return '\n'.join(doc) +def avail_easyconfig_params_json(): + """ + Compose overview of available easyconfig parameters, in json format. + """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json") + + def avail_easyconfig_params_txt(title, grouped_params): """ Compose overview of available easyconfig parameters, in plain text format. @@ -427,6 +451,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_templates_%s' % output_format, []) +def avail_easyconfig_templates_json(): + """ Returns template documentation in json text format """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates") + + def avail_easyconfig_templates_txt(): """ Returns template documentation in plain text format """ # This has to reflect the methods/steps used in easyconfig _generate_template_values @@ -641,6 +670,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT): + if output_format == FORMAT_JSON: + raise NotImplementedError("JSON output format not supported for list_easyblocks") format_strings = { FORMAT_MD: { 'det_root_templ': "- **%s** (%s%s)", @@ -1025,6 +1056,38 @@ def list_software_txt(software, detailed=False): return '\n'.join(lines) +def list_software_json(software, detailed=False): + """ + Return overview of supported software in json + + :param software: software information (strucuted like list_software does) + :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) + :return: multi-line string presenting requested info + """ + lines = ['['] + for key in sorted(software, key=lambda x: x.lower()): + for entry in software[key]: + if detailed: + # deep copy here to avoid modifying the original dict + entry = copy.deepcopy(entry) + entry['description'] = ' '.join(entry['description'].split('\n')).strip() + else: + entry = {} + entry['name'] = key + + lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",") + if not detailed: + break + + # remove trailing comma on last line + if len(lines) > 1: + lines[-1] = lines[-1].rstrip(',') + + lines.append(']') + + return '\n'.join(lines) + + def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') @@ -1173,6 +1236,11 @@ def list_toolchains_txt(tcs): return '\n'.join(doc) +def list_toolchains_json(tcs): + """ Returns overview of all toolchains in json format """ + raise NotImplementedError("JSON output not implemented yet for --list-toolchains") + + def avail_toolchain_opts(name, output_format=FORMAT_TXT): """Show list of known options for given toolchain.""" tc_class, _ = search_toolchain(name) @@ -1226,6 +1294,11 @@ def avail_toolchain_opts_rst(name, tc_dict): return '\n'.join(doc) +def avail_toolchain_opts_json(name, tc_dict): + """ Returns overview of toolchain options in jsonformat """ + raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts") + + def avail_toolchain_opts_txt(name, tc_dict): """ Returns overview of toolchain options in txt format """ doc = ["Available options for %s toolchain:" % name] @@ -1252,6 +1325,13 @@ def get_easyblock_classes(package_name): return easyblocks +def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None): + """ + Compose overview of all easyblocks in the given package in json format + """ + raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview") + + def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in MarkDown format diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index aa0b59c4ae..253dce094a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.configobj import ConfigObj, ConfigObjError -from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT +from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains @@ -466,7 +466,8 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]), + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, + [FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), diff --git a/test/framework/docs.py b/test/framework/docs.py index 84d862bd3c..70280892e4 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -405,6 +405,104 @@ ``1.4``|``GCC/4.6.3``, ``system`` ``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +LIST_SOFTWARE_SIMPLE_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + +* GCC +* gzip""" + +LIST_SOFTWARE_DETAILED_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + + +[GCC](#gcc) - [gzip](#gzip) + + +### GCC + +%(gcc_descr)s + +*homepage*: + +version |toolchain +---------|---------- +``4.6.3``|``system`` + +### gzip + +%(gzip_descr)s + +*homepage*: + +version|toolchain +-------|------------------------------- +``1.4``|``GCC/4.6.3``, ``system`` +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + +LIST_SOFTWARE_SIMPLE_JSON = """[ +{ + "name": "GCC" +}, +{ + "name": "gzip" +} +]""" + +LIST_SOFTWARE_DETAILED_JSON = """[ +{ + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "name": "GCC", + "toolchain": "system", + "version": "4.6.3", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "GCC/4.6.3", + "version": "1.4", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "system", + "version": "1.4", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "foss/2018a", + "version": "1.5", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "intel/2018a", + "version": "1.5", + "versionsuffix": "" +} +]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + class DocsTest(EnhancedTestCase): @@ -541,6 +639,9 @@ def test_license_docs(self): regex = re.compile(r"^``GPLv3``\s*|The GNU General Public License", re.M) self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs)) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, avail_easyconfig_licenses, output_format='json') + def test_list_easyblocks(self): """ Tests for list_easyblocks function @@ -569,6 +670,9 @@ def test_list_easyblocks(self): txt = list_easyblocks(list_easyblocks='detailed', output_format='md') self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_MD % {'topdir': topdir_easyblocks}) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, list_easyblocks, output_format='json') + def test_list_software(self): """Test list_software* functions.""" build_options = { @@ -587,6 +691,9 @@ def test_list_software(self): self.assertEqual(list_software(output_format='md'), LIST_SOFTWARE_SIMPLE_MD) self.assertEqual(list_software(output_format='md', detailed=True), LIST_SOFTWARE_DETAILED_MD) + self.assertEqual(list_software(output_format='json'), LIST_SOFTWARE_SIMPLE_JSON) + self.assertEqual(list_software(output_format='json', detailed=True), LIST_SOFTWARE_DETAILED_JSON) + # GCC/4.6.3 is installed, no gzip module installed txt = list_software(output_format='txt', detailed=True, only_installed=True) self.assertTrue(re.search(r'^\* GCC', txt, re.M)) @@ -690,6 +797,10 @@ def test_list_toolchains(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + list_toolchains(output_format='json') + def test_avail_cfgfile_constants(self): """ Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file. @@ -734,6 +845,10 @@ def test_avail_cfgfile_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='json') + def test_avail_easyconfig_constants(self): """ Test avail_easyconfig_constants to generate overview of constants that can be used in easyconfig files. @@ -777,6 +892,10 @@ def test_avail_easyconfig_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_constants(output_format='json') + def test_avail_easyconfig_templates(self): """ Test avail_easyconfig_templates to generate overview of templates that can be used in easyconfig files. @@ -827,6 +946,10 @@ def test_avail_easyconfig_templates(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_templates(output_format='json') + def test_avail_toolchain_opts(self): """ Test avail_toolchain_opts to generate overview of supported toolchain options. @@ -911,6 +1034,12 @@ def test_avail_toolchain_opts(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('foss', output_format='json') + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('intel', output_format='json') + def test_mk_table(self): """ Tests for mk_*_table functions.