From c8ed201285327104f56716c8c620a90f35f26364 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 20 Feb 2021 18:53:23 +0100 Subject: [PATCH] include %(mpi_cmd_prefix)s and %(cuda_*)s templates in output of --avail-easyconfig-templates (fixes #3585) --- easybuild/framework/easyconfig/templates.py | 25 ++++++++++- easybuild/tools/docs.py | 17 ++++++- test/framework/easyconfig.py | 1 + test/framework/options.py | 49 +++++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 38619c647f..a1ef4d42b7 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -46,7 +46,6 @@ # derived from easyconfig, but not from ._config directly TEMPLATE_NAMES_EASYCONFIG = [ - ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), ('module_name', "Module name"), ('nameletter', "First letter of software name"), ('toolchain_name', "Toolchain name"), @@ -86,6 +85,18 @@ ('Python', 'py'), ('R', 'r'), ] +# template values which are only generated dynamically +TEMPLATE_NAMES_DYNAMIC = [ + ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), + ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " + "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), + ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), + ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), + ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), + ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), +] + # constant templates that can be used in easyconfigs TEMPLATE_CONSTANTS = [ # source url constants @@ -300,6 +311,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) except Exception: _log.warning("Failed to get .lower() for name %s value %s (type %s)", name, value, type(value)) + # keep track of names of defined templates until now, + # so we can check whether names of additional dynamic template values are all known + common_template_names = set(template_values.keys()) + # step 5. add additional conditional templates if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'): try: @@ -322,6 +337,14 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) + unknown_names = [] + for key in template_values: + dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC) + if not (key in common_template_names or key in dynamic_template_names): + unknown_names.append(key) + if unknown_names: + raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names)) + return template_values diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index a052e176e2..3036a66a92 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -46,9 +46,9 @@ from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import EasyConfigParser -from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_EASYCONFIG +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC +from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE -from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_CONSTANTS from easybuild.framework.easyconfig.templates import TEMPLATE_SOFTWARE_VERSIONS, template_constant_dict from easybuild.framework.easyconfig.tools import avail_easyblocks from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs @@ -344,6 +344,12 @@ def avail_easyconfig_templates_txt(): for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + # some template values are only defined dynamically, + # see template_constant_dict function in easybuild.framework.easyconfigs.templates + doc.append('Template values which are defined dynamically') + for name in TEMPLATE_NAMES_DYNAMIC: + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + doc.append('Template constants that can be used in easyconfigs') for cst in TEMPLATE_CONSTANTS: doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1])) @@ -395,6 +401,13 @@ def avail_easyconfig_templates_rst(): ] doc.extend(rst_title_and_table(title, table_titles, table_values)) + title = 'Template values which are defined dynamically' + table_values = [ + ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], + [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ] + doc.extend(rst_title_and_table(title, table_titles, table_values)) + title = 'Template constants that can be used in easyconfigs' titles = ['Constant', 'Template value', 'Template name'] table_values = [ diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4b4a6a757f..d4011f124f 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1128,6 +1128,7 @@ def test_templating_doc(self): easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, + easyconfig.templates.TEMPLATE_NAMES_DYNAMIC, easyconfig.templates.TEMPLATE_CONSTANTS, ] diff --git a/test/framework/options.py b/test/framework/options.py index 9efd0ed240..dedd242068 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -509,6 +509,55 @@ def run_test(fmt=None): for fmt in [None, 'txt', 'rst']: run_test(fmt=fmt) + def test_avail_easyconfig_templates(self): + """Test listing available easyconfig file templates.""" + + def run_test(fmt=None): + """Helper function to test --avail-easyconfig-templates.""" + + args = ['--avail-easyconfig-templates'] + if fmt is not None: + args.append('--output-format=%s' % fmt) + + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(args, verbose=True, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + + if fmt == 'rst': + pattern_lines = [ + r'^``%\(version_major\)s``\s+Major version\s*$', + r'^``%\(cudaver\)s``\s+full version for CUDA\s*$', + r'^``%\(pyshortver\)s``\s+short version for Python \(.\)\s*$', + r'^\* ``%\(name\)s``$', + r'^``%\(namelower\)s``\s+lower case of value of name\s*$', + r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$', + r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$', + r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$', + ] + else: + pattern_lines = [ + r'^\s+%\(version_major\)s: Major version$', + r'^\s+%\(cudaver\)s: full version for CUDA$', + r'^\s+%\(pyshortver\)s: short version for Python \(.\)$', + r'^\s+%\(name\)s$', + r'^\s+%\(namelower\)s: lower case of value of name$', + r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$', + r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$', + r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$', + ] + + for pattern_line in pattern_lines: + regex = re.compile(pattern_line, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should match in: %s" % (regex.pattern, stdout)) + + for fmt in [None, 'txt', 'rst']: + run_test(fmt=fmt) + def test_avail_easyconfig_params(self): """Test listing available easyconfig parameters."""