diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b71f8bae59..2bea83469f 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3187,21 +3187,24 @@ def post_processing_step(self): ) return self.post_install_step() + def _dispatch_sanity_check_step(self, *args, **kwargs): + """Decide whether to run the dry-run or the real version of the sanity-check step""" + if self.dry_run: + self._sanity_check_step_dry_run(*args, **kwargs) + else: + self._sanity_check_step(*args, **kwargs) + def sanity_check_step(self, *args, **kwargs): """ Do a sanity check on the installation - if *any* of the files/subdirectories in the installation directory listed in sanity_check_paths are non-existent (or empty), the sanity check fails """ - if self.dry_run: - self._sanity_check_step_dry_run(*args, **kwargs) - # handling of extensions that were installed for multiple dependency versions is done in ExtensionEasyBlock - elif self.cfg['multi_deps'] and not self.is_extension: + if self.cfg['multi_deps'] and not self.is_extension: self._sanity_check_step_multi_deps(*args, **kwargs) - else: - self._sanity_check_step(*args, **kwargs) + self._dispatch_sanity_check_step(*args, **kwargs) def _sanity_check_step_multi_deps(self, *args, **kwargs): """Perform sanity check for installations that iterate over a list a versions for particular dependencies.""" @@ -3233,7 +3236,7 @@ def _sanity_check_step_multi_deps(self, *args, **kwargs): self.log.info(info_msg) kwargs['extra_modules'] = extra_modules - self._sanity_check_step(*args, **kwargs) + self._dispatch_sanity_check_step(*args, **kwargs) # restore list of lists of build dependencies & stop iterating again self.cfg['builddependencies'] = builddeps @@ -3509,7 +3512,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands): # if no sanity_check_paths are specified in easyconfig, # we fall back to the ones provided by the easyblock via custom_paths if custom_paths: - paths = custom_paths + paths = self.cfg.resolve_template(custom_paths) self.log.info("Using customized sanity check paths: %s", paths) # if custom_paths is empty, we fall back to a generic set of paths: # non-empty bin/ + /lib or /lib64 directories @@ -3523,14 +3526,13 @@ def _sanity_check_step_common(self, custom_paths, custom_commands): # if enhance_sanity_check is enabled *and* sanity_check_paths are specified in the easyconfig, # those paths are used to enhance the paths provided by the easyblock if enhance_sanity_check and ec_paths: - for key in ec_paths: - val = ec_paths[key] + for key, val in ec_paths.items(): if isinstance(val, list): paths[key] = paths.get(key, []) + val else: - error_pattern = "Incorrect value type in sanity_check_paths, should be a list: " - error_pattern += "%s (type: %s)" % (val, type(val)) - raise EasyBuildError(error_pattern) + error_msg = "Incorrect value type in sanity_check_paths, should be a list: " + error_msg += "%s (type: %s)" % (val, type(val)) + raise EasyBuildError(error_msg) self.log.info("Enhanced sanity check paths after taking into account easyconfig file: %s", paths) sorted_keys = sorted(paths.keys()) @@ -3560,7 +3562,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands): self.log.info("Using (only) sanity check commands specified by easyconfig file: %s", commands) else: if custom_commands: - commands = custom_commands + commands = self.cfg.resolve_template(custom_commands) self.log.info("Using customised sanity check commands: %s", commands) else: commands = [] diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index c95ab7dafe..08505316a5 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1769,6 +1769,12 @@ def _generate_template_values(self, ignore=None): if self.template_values[key] is None: del self.template_values[key] + def resolve_template(self, value): + """Resolve all templates in the given value using this easyconfig""" + if not self.template_values: + self.generate_template_values() + return resolve_template(value, self.template_values) + @handle_deprecated_or_replaced_easyconfig_parameters def __contains__(self, key): """Check whether easyconfig parameter is defined""" @@ -1784,9 +1790,7 @@ def __getitem__(self, key): raise EasyBuildError("Use of unknown easyconfig parameter '%s' when getting parameter value", key) if self.enable_templating: - if self.template_values is None or len(self.template_values) == 0: - self.generate_template_values() - value = resolve_template(value, self.template_values) + value = self.resolve_template(value) return value diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 0171d74f10..6c76d9715c 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -198,10 +198,8 @@ def include_easyblocks(tmpdir, paths): # hard inject location to included (generic) easyblocks into Python search path # only prepending to sys.path is not enough due to 'pkgutil.extend_path' in easybuild/easyblocks/__init__.py - new_path = os.path.join(easyblocks_path, 'easybuild', 'easyblocks') - easybuild.easyblocks.__path__.insert(0, new_path) - new_path = os.path.join(new_path, 'generic') - easybuild.easyblocks.generic.__path__.insert(0, new_path) + easybuild.easyblocks.__path__.insert(0, easyblocks_dir) + easybuild.easyblocks.generic.__path__.insert(0, os.path.join(easyblocks_dir, 'generic')) # sanity check: verify that included easyblocks can be imported (from expected location) for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]: diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index e8cd07de4a..e4afeadc79 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('5.0.0.dev0') +VERSION = LooseVersion('5.0.0beta1') UNKNOWN = 'UNKNOWN' UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4661344d8d..e8b1dd0cb7 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1279,6 +1279,29 @@ def test_template_constant_import(self): self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0]) self.assertEqual(SHLIB_EXT, get_shared_lib_ext()) + def test_ec_method_resolve_template(self): + """Test the `resolve_template` method of easyconfig instances.""" + # don't use any escaping insanity here, since it is templated itself + self.contents = textwrap.dedent(""" + easyblock = "ConfigureMake" + name = "PI" + version = "3.14" + homepage = "http://example.com" + description = "test easyconfig %(name)s version %(version_major)s" + toolchain = SYSTEM + """) + self.prep() + ec = EasyConfig(self.eb_file, validate=False) + + # We can resolve anything with values from the EC + self.assertEqual(ec.resolve_template('%(namelower)s %(version_major)s begins with %(nameletterlower)s'), + 'pi 3 begins with p') + + # `resolve_template` does basically the same resolving any value on acccess + description = ec.get('description', resolve=False) + self.assertIn('%', description, 'Description needs a template for the next test') + self.assertEqual(ec.resolve_template(description), ec['description']) + def test_templating_cuda_toolchain(self): """Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain.""" diff --git a/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.15.eb b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.15.eb new file mode 100644 index 0000000000..b9f408e3ef --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.15.eb @@ -0,0 +1,23 @@ +easyblock = 'ConfigureMake' + +name = 'Python' +version = '2.7.15' + +homepage = 'http://python.org/' +description = """Python is a programming language that lets you work more quickly and integrate your systems +more effectively.""" + +toolchain = SYSTEM + +source_urls = ['http://www.python.org/ftp/%(namelower)s/%(version)s/'] +sources = [SOURCE_TGZ] + +# This just serves to have a Python as a dependency to test e.g. the Python version templates +# So all dependencies and extensions are removed +dependencies = [] + +osdependencies = [] + +exts_list = [] + +moduleclass = 'lang' diff --git a/test/framework/easyconfigs/test_ecs/p/Python/Python-3.7.2.eb b/test/framework/easyconfigs/test_ecs/p/Python/Python-3.7.2.eb new file mode 100644 index 0000000000..a61cabbc32 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/p/Python/Python-3.7.2.eb @@ -0,0 +1,23 @@ +easyblock = 'ConfigureMake' + +name = 'Python' +version = '3.7.2' + +homepage = 'http://python.org/' +description = """Python is a programming language that lets you work more quickly and integrate your systems +more effectively.""" + +toolchain = SYSTEM + +source_urls = ['http://www.python.org/ftp/%(namelower)s/%(version)s/'] +sources = [SOURCE_TGZ] + +# This just serves to have a Python as a dependency to test e.g. the Python version templates +# So all dependencies and extensions are removed +dependencies = [] + +osdependencies = [] + +exts_list = [] + +moduleclass = 'lang' diff --git a/test/framework/filetools.py b/test/framework/filetools.py index e59c3099f1..2f5f6bfa7f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2545,7 +2545,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 92) + self.assertEqual(len(index), 94) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/sandbox/sources/p/Python/Python-3.7.2.tgz b/test/framework/sandbox/sources/p/Python/Python-3.7.2.tgz new file mode 100644 index 0000000000..a1583ec2c0 Binary files /dev/null and b/test/framework/sandbox/sources/p/Python/Python-3.7.2.tgz differ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index b23eded3e2..f76bb6942a 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2673,6 +2673,99 @@ def test_toy_build_enhanced_sanity_check(self): del sys.modules['easybuild.easyblocks.toy'] + def test_toy_build_enhanced_sanity_check_templated_multi_dep(self): + """Test enhancing of sanity check by easyblocks with templates and in the presence of multi_deps.""" + + # if toy easyblock was imported, get rid of corresponding entry in sys.modules, + # to avoid that it messes up the use of --include-easyblocks=toy.py below... + if 'easybuild.easyblocks.toy' in sys.modules: + del sys.modules['easybuild.easyblocks.toy'] + + test_dir = os.path.join(os.path.abspath(os.path.dirname(__file__))) + toy_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + + # get rid of custom sanity check paths in test easyconfig + regex = re.compile(r'^sanity_check_paths\s*=\s*{[^}]+}', re.M) + test_ec_txt = regex.sub('', toy_ec_txt) + self.assertNotIn('sanity_check_', test_ec_txt) + + test_ec_txt += "\nmulti_deps = {'Python': ['3.7.2', '2.7.15']}" + write_file(test_ec, test_ec_txt) + + # create custom easyblock for toy that has a custom sanity_check_step + toy_easyblock = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + + toy_easyblock_txt = read_file(toy_easyblock) + + toy_custom_sanity_check_step = textwrap.dedent(""" + # Add to class to indent + def sanity_check_step(self): + paths = { + 'files': ['bin/python%(pyshortver)s'], + 'dirs': ['lib/py-%(pyshortver)s'], + } + cmds = ['python%(pyshortver)s'] + return super(EB_toy, self).sanity_check_step(custom_paths=paths, custom_commands=cmds) + """) + test_toy_easyblock = os.path.join(self.test_prefix, 'toy.py') + write_file(test_toy_easyblock, toy_easyblock_txt + toy_custom_sanity_check_step) + + eb_args = [ + '--extended-dry-run', + '--include-easyblocks=%s' % test_toy_easyblock, + ] + + # by default, sanity check commands & paths specified by easyblock are used + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True) + stdout = self.get_stdout() + # Cut output to start of the toy-ec, after the Python installations + stdout = stdout[stdout.index(test_ec):] + + pattern_template = textwrap.dedent(r""" + Sanity check paths - file.* + \s*\* bin/python{pyshortver} + Sanity check paths - \(non-empty\) directory.* + \s*\* lib/py-{pyshortver} + Sanity check commands + \s*\* python{pyshortver} + """) + for pyshortver in ('2.7', '3.7'): + regex = re.compile(pattern_template.format(pyshortver=pyshortver), re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + # Enhance sanity check by extra paths to check for, the ones from the easyblock should be kept + test_ec_txt += textwrap.dedent(""" + enhance_sanity_check = True + sanity_check_paths = { + 'files': ['bin/pip%(pyshortver)s'], + 'dirs': ['bin'], + } + """) + write_file(test_ec, test_ec_txt) + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True) + stdout = self.get_stdout() + # Cut output to start of the toy-ec, after the Python installations + stdout = stdout[stdout.index(test_ec):] + + pattern_template = textwrap.dedent(r""" + Sanity check paths - file.* + \s*\* bin/pip{pyshortver} + \s*\* bin/python{pyshortver} + Sanity check paths - \(non-empty\) directory.* + \s*\* bin + \s*\* lib/py-{pyshortver} + Sanity check commands + \s*\* python{pyshortver} + """) + for pyshortver in ('2.7', '3.7'): + regex = re.compile(pattern_template.format(pyshortver=pyshortver), re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_toy_dumped_easyconfig(self): """ Test dumping of file in eb_filerepo in both .eb format """ filename = 'toy-0.0'