From 4f8ddb824b346011982a9aa62d9698839a1d49cc Mon Sep 17 00:00:00 2001 From: Rovanion Luckey Date: Wed, 19 Apr 2023 14:20:56 +0200 Subject: [PATCH 001/171] Build reproducible tarballs from git --- easybuild/tools/filetools.py | 14 ++++--- test/framework/filetools.py | 72 +++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 80a6ba6560..7a431c8ee5 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2690,11 +2690,15 @@ def get_source_tarball_from_git(filename, targetdir, git_config): for cmd in cmds: run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name, trace=False) - # create an archive and delete the git repo directory - if keep_git_dir: - tar_cmd = ['tar', 'cfvz', targetpath, repo_name] - else: - tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] + # When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag + # '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and + # --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/ + tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|', + 'LC_ALL=C', 'sort', '--zero-terminated', '|', + 'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion', + '--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0', + '--numeric-owner', '--format=gnu', '--null', + '--no-recursion', '--files-from -'] run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False, trace=False) # cleanup (repo_name dir does not exist in dry run mode) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index fcaebe16d4..5e2d0412fe 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2787,41 +2787,57 @@ def run_check(): 'url': 'git@github.com:easybuilders', 'tag': 'tag_for_tests', } - git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter + string_args = { + 'git_repo': 'git@github.com:easybuilders/testrepository.git', + 'test_prefix': self.test_prefix, + } + expected = '\n'.join([ r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() del git_config['clone_into'] git_config['recursive'] = True expected = '\n'.join([ r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() git_config['keep_git_dir'] = True expected = '\n'.join([ r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', - r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz testrepository"', - r" \(in /.*\)", - ]) % git_repo + r" \(in .*/tmp.*\)", + r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar' + r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip' + r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion' + r' --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() del git_config['keep_git_dir'] @@ -2829,23 +2845,29 @@ def run_check(): git_config['commit'] = '8456f86' expected = '\n'.join([ r' running command "git clone --no-checkout %(git_repo)s"', - r" \(in /.*\)", + r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() del git_config['recursive'] expected = '\n'.join([ r' running command "git clone --no-checkout %(git_repo)s"', - r" \(in /.*\)", + r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - r" \(in /.*\)", - ]) % git_repo + r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' + r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' + r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' + r' --null --no-recursion --files-from -"', + r" \(in .*/tmp.*\)", + ]) % string_args run_check() # Test with real data. From 0700f2dc7fdcf66e39533cf1478c05540da982ea Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:53:44 +0100 Subject: [PATCH 002/171] rename Extension.run() method to install_extension() --- easybuild/framework/easyblock.py | 4 ++-- easybuild/framework/extension.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 4 ++-- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 52a700bdfe..622e806094 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1914,12 +1914,12 @@ def install_extensions_sequential(self, install=True): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) - # real work + # actual installation of the extension if install: try: ext.prerun() with self.module_generator.start_module_creation(): - txt = ext.run() + txt = ext.install_extension() if txt: self.module_extra_extensions += txt ext.postrun() diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 8dc4669392..dac48636ef 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -177,7 +177,7 @@ def prerun(self): """ pass - def run(self, *args, **kwargs): + def install_extension(self, *args, **kwargs): """ Actual installation of an extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 9c700cf779..42f56d6478 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -59,7 +59,7 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def run(self, *args, **kwargs): + def install_extension(self, *args, **kwargs): """ Install toy extension. """ @@ -78,7 +78,7 @@ def prerun(self): super(Toy_Extension, self).prerun() if self.src: - super(Toy_Extension, self).run(unpack_src=True) + super(Toy_Extension, self).install_extension(unpack_src=True) EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) def run_async(self): diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index c4614e3333..b361df126b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -154,10 +154,10 @@ def prerun(self): """ Prepare installation of toy as extension. """ - super(EB_toy, self).run(unpack_src=True) + super(EB_toy, self).install_extension(unpack_src=True) self.configure_step() - def run(self): + def install_extension(self): """ Install toy as extension. """ From 202c78ed725dea5fd389a693ccb93866728e1569 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 15:30:54 +0100 Subject: [PATCH 003/171] rename ExtensionEasyblock.run() to install_extension()" --- easybuild/framework/extensioneasyblock.py | 2 +- test/framework/easyblock.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 6ad116b20b..c06fb56e0f 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -133,7 +133,7 @@ def _set_start_dir(self): self.log.warning(warn_msg) print_warning(warn_msg, silent=build_option('silent')) - def run(self, unpack_src=False): + def install_extension(self, unpack_src=False): """Common operations for extensions: unpacking sources, patching, ...""" # unpack file if desired diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 9f656573d1..ae1ebcd56d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2207,7 +2207,7 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): eb.extensions_step(fetch=True, install=False) # extract sources of the extension ext = eb.ext_instances[-1] - ext.run(unpack_src=unpack_src) + ext.install_extension(unpack_src=unpack_src) if expected_start_dir is None: self.assertIsNone(ext.start_dir) From d9cb50e8618421a67aa5ea3a4ac31a0747cd7c20 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:56:05 +0100 Subject: [PATCH 004/171] rename Extension.prerun() method to preinstall_extension() --- easybuild/framework/easyblock.py | 4 ++-- easybuild/framework/extension.py | 2 +- .../easyblocks/generic/toy_extension.py | 20 +++++++++---------- .../sandbox/easybuild/easyblocks/t/toy.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 622e806094..85cccc6baf 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1917,7 +1917,7 @@ def install_extensions_sequential(self, install=True): # actual installation of the extension if install: try: - ext.prerun() + ext.preinstall_extension() with self.module_generator.start_module_creation(): txt = ext.install_extension() if txt: @@ -2054,7 +2054,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) if install: - ext.prerun() + ext.preinstall_extension() ext.run_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index dac48636ef..2330ab348f 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -171,7 +171,7 @@ def version(self): """ return self.ext.get('version', None) - def prerun(self): + def preinstall_extension(self): """ Stuff to do before installing a extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 42f56d6478..2a15a2cb3b 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -59,6 +59,16 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) + def preinstall_extension(self): + """ + Prepare installation of toy extension. + """ + super(Toy_Extension, self).preinstall_extension() + + if self.src: + super(Toy_Extension, self).install_extension(unpack_src=True) + EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) + def install_extension(self, *args, **kwargs): """ Install toy extension. @@ -71,16 +81,6 @@ def install_extension(self, *args, **kwargs): return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name) - def prerun(self): - """ - Prepare installation of toy extension. - """ - super(Toy_Extension, self).prerun() - - if self.src: - super(Toy_Extension, self).install_extension(unpack_src=True) - EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) - def run_async(self): """ Install toy extension asynchronously. diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index b361df126b..0a6ca8e7a8 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -150,7 +150,7 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def prerun(self): + def preinstall_extension(self): """ Prepare installation of toy as extension. """ From f85564b9c7ba6dfcc21784a9d24edaba1d15e126 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:57:54 +0100 Subject: [PATCH 005/171] rename Extension.postrun() method to postinstall_extension() --- easybuild/framework/easyblock.py | 4 ++-- easybuild/framework/extension.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 4 ++-- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 85cccc6baf..5d9c37cec4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1922,7 +1922,7 @@ def install_extensions_sequential(self, install=True): txt = ext.install_extension() if txt: self.module_extra_extensions += txt - ext.postrun() + ext.postinstall_extension() finally: if not self.dry_run: ext_duration = datetime.now() - start_time @@ -1983,7 +1983,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): for ext in running_exts[:]: if self.dry_run or ext.async_cmd_check(): self.log.info("Installation of %s completed!", ext.name) - ext.postrun() + ext.postinstall_extension() running_exts.remove(ext) installed_ext_names.append(ext.name) update_exts_progress_bar_helper(running_exts, 1) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 2330ab348f..b84ddf8d8b 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -189,7 +189,7 @@ def run_async(self, *args, **kwargs): """ raise NotImplementedError - def postrun(self): + def postinstall_extension(self): """ Stuff to do after installing a extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 2a15a2cb3b..07139c92ec 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -91,11 +91,11 @@ def run_async(self): else: self.async_cmd_info = False - def postrun(self): + def postinstall_extension(self): """ Wrap up installation of toy extension. """ - super(Toy_Extension, self).postrun() + super(Toy_Extension, self).postinstall_extension() EB_toy.install_step(self.master, name=self.name) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 0a6ca8e7a8..e84a40c1c7 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -170,7 +170,7 @@ def run_async(self): cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) self.async_cmd_start(cmd) - def postrun(self): + def postinstall_extension(self): """ Wrap up installation of toy as extension. """ From 23446dfb4521be967f2f797775f7396be102e042 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 09:59:14 +0100 Subject: [PATCH 006/171] rename Extension.run_async() method to install_extension_async() --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/extension.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5d9c37cec4..a6c111c29e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2055,7 +2055,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.preinstall_extension() - ext.run_async() + ext.install_extension_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index b84ddf8d8b..a9fff98020 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -183,7 +183,7 @@ def install_extension(self, *args, **kwargs): """ pass - def run_async(self, *args, **kwargs): + def install_extension_async(self, *args, **kwargs): """ Asynchronous installation of an extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 07139c92ec..204da5458f 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -81,7 +81,7 @@ def install_extension(self, *args, **kwargs): return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name) - def run_async(self): + def install_extension_async(self): """ Install toy extension asynchronously. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index e84a40c1c7..f87696f122 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -163,7 +163,7 @@ def install_extension(self): """ self.build_step() - def run_async(self): + def install_extension_async(self): """ Asynchronous installation of toy as extension. """ From 845d11085d83f128e58c92789331956d039da5d4 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 12 Dec 2023 15:25:37 +0100 Subject: [PATCH 007/171] rename EasyBlock.install_extensions() to install_all_extensions() --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a6c111c29e..62cbe9873d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1850,7 +1850,7 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances - def install_extensions(self, install=True): + def install_all_extensions(self, install=True): """ Install extensions. @@ -2883,7 +2883,7 @@ def extensions_step(self, fetch=False, install=True): if self.skip: self.skip_extensions() - self.install_extensions(install=install) + self.install_all_extensions(install=install) # cleanup (unload fake module, remove fake module dir) if fake_mod_data: From 6b00163e2ed6d674881f8b6261de0fff3cdf633c Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 13 Dec 2023 16:41:04 +0100 Subject: [PATCH 008/171] separate prefix in preinstall_extension and postinstall_extension method names --- easybuild/framework/easyblock.py | 8 ++++---- easybuild/framework/extension.py | 4 ++-- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 8 ++++---- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 62cbe9873d..fe45be97eb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1917,12 +1917,12 @@ def install_extensions_sequential(self, install=True): # actual installation of the extension if install: try: - ext.preinstall_extension() + ext.pre_install_extension() with self.module_generator.start_module_creation(): txt = ext.install_extension() if txt: self.module_extra_extensions += txt - ext.postinstall_extension() + ext.post_install_extension() finally: if not self.dry_run: ext_duration = datetime.now() - start_time @@ -1983,7 +1983,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): for ext in running_exts[:]: if self.dry_run or ext.async_cmd_check(): self.log.info("Installation of %s completed!", ext.name) - ext.postinstall_extension() + ext.post_install_extension() running_exts.remove(ext) installed_ext_names.append(ext.name) update_exts_progress_bar_helper(running_exts, 1) @@ -2054,7 +2054,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) if install: - ext.preinstall_extension() + ext.pre_install_extension() ext.install_extension_async() running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a9fff98020..e2cbe2c323 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -171,7 +171,7 @@ def version(self): """ return self.ext.get('version', None) - def preinstall_extension(self): + def pre_install_extension(self): """ Stuff to do before installing a extension. """ @@ -189,7 +189,7 @@ def install_extension_async(self, *args, **kwargs): """ raise NotImplementedError - def postinstall_extension(self): + def post_install_extension(self): """ Stuff to do after installing a extension. """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 204da5458f..34aa1eb259 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -59,11 +59,11 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def preinstall_extension(self): + def pre_install_extension(self): """ Prepare installation of toy extension. """ - super(Toy_Extension, self).preinstall_extension() + super(Toy_Extension, self).pre_install_extension() if self.src: super(Toy_Extension, self).install_extension(unpack_src=True) @@ -91,11 +91,11 @@ def install_extension_async(self): else: self.async_cmd_info = False - def postinstall_extension(self): + def post_install_extension(self): """ Wrap up installation of toy extension. """ - super(Toy_Extension, self).postinstall_extension() + super(Toy_Extension, self).post_install_extension() EB_toy.install_step(self.master, name=self.name) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index f87696f122..3ff4b73d67 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -150,7 +150,7 @@ def required_deps(self): else: raise EasyBuildError("Dependencies for %s are unknown!", self.name) - def preinstall_extension(self): + def pre_install_extension(self): """ Prepare installation of toy as extension. """ @@ -170,7 +170,7 @@ def install_extension_async(self): cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) self.async_cmd_start(cmd) - def postinstall_extension(self): + def post_install_extension(self): """ Wrap up installation of toy as extension. """ From 9d92bcea5f43eac5e0128ea3841a9fb2c3444533 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 13 Dec 2023 16:57:35 +0100 Subject: [PATCH 009/171] add deprecated extension install methods with deprecation warnings --- easybuild/framework/easyblock.py | 8 +++++++ easybuild/framework/extension.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index fe45be97eb..914e8ad4a0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1850,6 +1850,14 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances + def install_extensions(self, *args, **kwargs): + """[DEPRECATED] Install extensions.""" + self.log.deprecated( + "Easyblock.install_extensions() is deprecated, use Easyblock.install_all_extensions() instead.", + '5.0', + ) + self.install_all_extensions(*args, **kwargs) + def install_all_extensions(self, install=True): """ Install extensions. diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index e2cbe2c323..31cfb8d373 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -171,24 +171,64 @@ def version(self): """ return self.ext.get('version', None) + def prerun(self): + """ + [DEPRECATED] Stuff to do before installing a extension. + """ + self.log.deprecated( + "Extension.prerun() is deprecated, use Extension.pre_install_extension() instead.", + '5.0', + ) + self.pre_install_extension() + def pre_install_extension(self): """ Stuff to do before installing a extension. """ pass + def run(self, *args, **kwargs): + """ + [DEPRECATED] Actual installation of an extension. + """ + self.log.deprecated( + "Extension.run() is deprecated, use Extension.install_extension() instead.", + '5.0', + ) + self.install_extension(*args, **kwargs) + def install_extension(self, *args, **kwargs): """ Actual installation of an extension. """ pass + def run_async(self, *args, **kwargs): + """ + [DEPRECATED] Asynchronous installation of an extension. + """ + self.log.deprecated( + "Extension.run_async() is deprecated, use Extension.install_extension_async() instead.", + '5.0', + ) + self.install_extension_async(*args, **kwargs) + def install_extension_async(self, *args, **kwargs): """ Asynchronous installation of an extension. """ raise NotImplementedError + def postrun(self): + """ + [DEPRECATED] Stuff to do after installing a extension. + """ + self.log.deprecated( + "Extension.postrun() is deprecated, use Extension.post_install_extension() instead.", + '5.0', + ) + self.post_install_extension() + def post_install_extension(self): """ Stuff to do after installing a extension. From e50ffb940c95a412fa512c9a0c18f246511db4d0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 13:40:33 +0100 Subject: [PATCH 010/171] make `is_rpath_wrapper` faster Avoid uneccessary read of the FULL binary file if it cannot be an RPATH wrapper because it is in the wrong folder. --- easybuild/tools/toolchain/toolchain.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a16fe35dc0..17bc9e881f 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -941,10 +941,11 @@ def is_rpath_wrapper(path): """ Check whether command at specified location already is an RPATH wrapper script rather than the actual command """ - in_rpath_wrappers_dir = os.path.basename(os.path.dirname(os.path.dirname(path))) == RPATH_WRAPPERS_SUBDIR + if os.path.basename(os.path.dirname(os.path.dirname(path))) != RPATH_WRAPPERS_SUBDIR: + return False + # Check if `rpath_args`` is called in the file # need to use binary mode to read the file, since it may be an actual compiler command (which is a binary file) - calls_rpath_args = b'rpath_args.py $CMD' in read_file(path, mode='rb') - return in_rpath_wrappers_dir and calls_rpath_args + return b'rpath_args.py $CMD' in read_file(path, mode='rb') def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None): """ From e346d636ac448b71825c0d5a5350ac11bc7f0595 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Dec 2023 09:36:11 +0100 Subject: [PATCH 011/171] add terse support to `--missing-modules` It is useful to have the missing modules in a format that can be processed in a script, i.e. one EC name per line. So allow `--terse` in combination with `--missing-modules`. --- easybuild/main.py | 6 +++--- easybuild/tools/robot.py | 8 +++++--- test/framework/options.py | 24 ++++++++++++++++++------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..826880aad1 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -495,7 +495,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: if options.missing_modules: - txt = missing_deps(easyconfigs, modtool) + txt = missing_deps(easyconfigs, modtool, terse=options.terse) else: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) @@ -684,7 +684,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr options.list_prs, options.merge_pr, options.review_pr, - options.terse, + options.terse and not options.missing_modules, search_query, ] if any(early_stop_options): @@ -721,7 +721,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if do_cleanup: - cleanup(logfile, eb_tmpdir, testing, silent=False) + cleanup(logfile, eb_tmpdir, testing, silent=options.terse) def prepare_main(args=None, logfile=None, testing=None): diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..3a21bc2f7f 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -288,16 +288,18 @@ def dry_run(easyconfigs, modtool, short=False): return '\n'.join(lines) -def missing_deps(easyconfigs, modtool): +def missing_deps(easyconfigs, modtool, terse=False): """ Determine subset of easyconfigs for which no module is installed yet. """ ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False) missing = skip_available(ordered_ecs, modtool) - if missing: + if terse: + lines = [os.path.basename(x['ec'].path) for x in missing] + elif missing: lines = ['', "%d out of %d required modules missing:" % (len(missing), len(ordered_ecs)), ''] - for ec in [x['ec'] for x in missing]: + for ec in (x['ec'] for x in missing): if ec.short_mod_name != ec.full_mod_name: modname = '%s | %s' % (ec.mod_subdir, ec.short_mod_name) else: diff --git a/test/framework/options.py b/test/framework/options.py index e6be49ab4f..a246311727 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1472,14 +1472,26 @@ def test_missing(self): ]) for opt in ['-M', '--missing-modules']: - self.mock_stderr(True) - self.mock_stdout(True) - self.eb_main(args + [opt], testing=False, raise_error=True) - stderr, stdout = self.get_stderr(), self.get_stdout() - self.mock_stderr(False) - self.mock_stdout(False) + with self.mocked_stdout_stderr(): + self.eb_main(args + [opt], testing=False, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() self.assertFalse(stderr) self.assertIn(expected, stdout) + # --terse + with self.mocked_stdout_stderr(): + self.eb_main(args + ['-M', '--terse'], testing=False, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.assertFalse(stderr) + if mns == 'HierarchicalMNS': + expected = '\n'.join([ + "GCC-4.6.3.eb", + "intel-2018a.eb", + "toy-0.0-deps.eb", + "gzip-1.4-GCC-4.6.3.eb", + ]) + else: + expected = 'gzip-1.4-GCC-4.6.3.eb' + self.assertEqual(stdout, expected + '\n') def test_dry_run_short(self): """Test dry run (short format).""" From b1efd5ced3f46b2dad972ff3e0100d4962f454b5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Dec 2023 10:06:59 +0100 Subject: [PATCH 012/171] Don't print index status when using `--terse` --- easybuild/tools/config.py | 1 + easybuild/tools/filetools.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6c0a173fe4..f025ce831c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -306,6 +306,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'skip_test_cases', 'skip_test_step', 'sticky_bit', + 'terse', 'trace', 'unit_testing_mode', 'upload_test_report', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6e4bd2d83e..21d9d5e5e0 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -966,11 +966,12 @@ def load_index(path, ignore_dirs=None): # check whether index is still valid if valid_ts: curr_ts = datetime.datetime.now() + terse = build_option('terse') if curr_ts > valid_ts: - print_warning("Index for %s is no longer valid (too old), so ignoring it...", path) + print_warning("Index for %s is no longer valid (too old), so ignoring it...", path, silent=terse) index = None else: - print_msg("found valid index for %s, so using it...", path) + print_msg("found valid index for %s, so using it...", path, silent=terse) return index or None From b1c5d082e1940b81da947203779937edb10bb993 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 10:22:58 +0100 Subject: [PATCH 013/171] Remove superflous string formatting `"%s" % var` is the same as `str(var)` while the latter is faster due to less parsing, inserting etc. In some places `var` was already a string so it can just be used. --- easybuild/base/generaloption.py | 8 ++++---- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/tools/docs.py | 6 +++--- easybuild/tools/filetools.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/testing.py | 2 +- test/framework/module_generator.py | 8 ++++---- test/framework/options.py | 2 +- .../module_naming_scheme/test_module_naming_scheme.py | 4 ++-- test/framework/toy_build.py | 4 ++-- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..6a4c5f0d5d 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -90,7 +90,7 @@ def set_columns(cols=None): pass if cols is not None: - os.environ['COLUMNS'] = "%s" % cols + os.environ['COLUMNS'] = str(cols) def what_str_list_tuple(name): @@ -822,8 +822,8 @@ def get_env_options(self): self.environment_arguments.append("%s=%s" % (lo, val)) else: # interpretation of values: 0/no/false means: don't set it - if ("%s" % val).lower() not in ("0", "no", "false",): - self.environment_arguments.append("%s" % lo) + if str(val).lower() not in ("0", "no", "false",): + self.environment_arguments.append(str(lo)) else: self.log.debug("Environment variable %s is not set" % env_opt_name) @@ -1189,7 +1189,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non for extra_detail in details[4:]: if isinstance(extra_detail, (list, tuple,)): # choices - nameds['choices'] = ["%s" % x for x in extra_detail] # force to strings + nameds['choices'] = [str(x) for x in extra_detail] # force to strings hlp += ' (choices: %s)' % ', '.join(nameds['choices']) elif isinstance(extra_detail, string_type) and len(extra_detail) == 1: args.insert(0, "-%s" % extra_detail) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 8abb697ece..9a1626c60b 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -370,7 +370,7 @@ def parse(self, configobj): for key, value in self.supported.items(): if key not in self.VERSION_OPERATOR_VALUE_TYPES: raise EasyBuildError('Unsupported key %s in %s section', key, self.SECTION_MARKER_SUPPORTED) - self.sections['%s' % key] = value + self.sections[key] = value for key, supported_key, fn_name in [('version', 'versions', 'get_version_str'), ('toolchain', 'toolchains', 'as_dict')]: diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c3e26ab868..7aaa69b69a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -324,7 +324,7 @@ def __repr__(self): newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: - newval = "%s" % fval + newval = str(fval) _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..4fcf1ad466 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -289,7 +289,7 @@ def avail_easyconfig_licenses_md(): lics = sorted(EASYCONFIG_LICENSES_DICT.items()) table_values = [ ["``%s``" % lic().name for _, lic in lics], - ["%s" % lic().description for _, lic in lics], + [lic().description for _, lic in lics], ["``%s``" % lic().version for _, lic in lics], ] @@ -1199,7 +1199,7 @@ def avail_toolchain_opts_md(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] @@ -1217,7 +1217,7 @@ def avail_toolchain_opts_rst(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6e4bd2d83e..5a2f724bc8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2668,7 +2668,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.append('%s/%s.git' % (url, repo_name)) if clone_into: - clone_cmd.append('%s' % clone_into) + clone_cmd.append(clone_into) tmpdir = tempfile.mkdtemp() cwd = change_dir(tmpdir) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0c19b93c8b..45582bbe5c 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1289,7 +1289,7 @@ def get_description(self, conflict=True): extensions_list = self._generate_extensions_list() if extensions_list: - extensions_stmt = 'extensions("%s")' % ','.join(['%s' % x for x in extensions_list]) + extensions_stmt = 'extensions("%s")' % ','.join([str(x) for x in extensions_list]) # put this behind a Lmod version check as 'extensions' is only (well) supported since Lmod 8.2.8, # see https://lmod.readthedocs.io/en/latest/330_extensions.html#module-extensions and # https://github.com/TACC/Lmod/issues/428 diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 68eedc23ce..ffd8ce580b 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -178,7 +178,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ ]) test_report.extend([ "#### Test result", - "%s" % msg, + msg, "", ]) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index a0c6131140..d2fbb6f642 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -82,7 +82,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -107,7 +107,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -137,7 +137,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -161,7 +161,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", diff --git a/test/framework/options.py b/test/framework/options.py index f3173dbb87..3bfc863e41 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2190,7 +2190,7 @@ def test_ignore_osdeps(self): self.assertTrue(regex.search(outtxt), "OS dependencies are checked, outtxt: %s" % outtxt) msg = "One or more OS dependencies were not found: " msg += r"\[\('nosuchosdependency',\), \('nosuchdep_option1', 'nosuchdep_option2'\)\]" - regex = re.compile(r'%s' % msg, re.M) + regex = re.compile(msg, re.M) self.assertTrue(regex.search(outtxt), "OS dependencies are honored, outtxt: %s" % outtxt) # check whether OS dependencies are effectively ignored diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 5eb5f66682..02934e5745 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -64,6 +64,6 @@ def det_module_symlink_paths(self, ec): def is_short_modname_for(self, modname, name): """ - Determine whether the specified (short) module name is a module for software with the specified name. + Determine whether the specified (short) module name is a moduleq for software with the specified name. """ - return modname.find('%s' % name) != -1 + return modname.find(name) != -1 diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9ccb3c08df..cc84ce86d1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1491,7 +1491,7 @@ def test_toy_module_fulltxt(self): mod_txt_regex_pattern = '\n'.join([ r'help\(\[==\[', r'', - r'%s' % help_txt, + help_txt, r'\]==\]\)', r'', r'whatis\(\[==\[Description: Toy C program, 100% toy.\]==\]\)', @@ -1528,7 +1528,7 @@ def test_toy_module_fulltxt(self): r'proc ModulesHelp { } {', r' puts stderr {', r'', - r'%s' % help_txt, + help_txt, r' }', r'}', r'', From 116e6db20a1dec9014fea562b157e97cfed0b08e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 09:28:03 +0100 Subject: [PATCH 014/171] Cleanup & Speedup environment checks `os.environ` is a dict hence the check for existance and access can usually be combined into a single access of the key avoiding the repeated lookup. Especially for `os.getenv` which already returns `None` when the key doesn't exist. Also use `.pop(key, None)` to remove a key without checking for it's existance first to avoid C&P errors --- easybuild/base/fancylogger.py | 15 +++++++-------- easybuild/main.py | 3 +-- easybuild/tools/environment.py | 4 ++-- easybuild/tools/modules.py | 12 ++++-------- test/framework/environment.py | 3 +-- test/framework/filetools.py | 6 ++---- test/framework/module_generator.py | 3 +-- test/framework/modules.py | 12 ++++-------- test/framework/modulestool.py | 3 +-- test/framework/options.py | 12 ++++-------- test/framework/robot.py | 3 +-- test/framework/toolchain.py | 12 ++++-------- test/framework/toy_build.py | 3 +-- test/framework/utilities.py | 6 ++---- 14 files changed, 35 insertions(+), 62 deletions(-) diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index b5a63477c6..8b5f0a8178 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -147,10 +147,10 @@ def _env_to_boolean(varname, default=False): >>> _env_to_boolean('NO_FOOBAR') False """ - if varname not in os.environ: + try: + return os.environ[varname].lower() in ('1', 'yes', 'true', 'y') + except KeyError: return default - else: - return os.environ.get(varname).lower() in ('1', 'yes', 'true', 'y') OPTIMIZED_ANSWER = "not available in optimized mode" @@ -911,16 +911,15 @@ def resetroot(): _default_logTo = None if 'FANCYLOG_SERVER' in os.environ: server = os.environ['FANCYLOG_SERVER'] - port = DEFAULT_UDP_PORT if ':' in server: server, port = server.split(':') + else: + port = DEFAULT_UDP_PORT # maybe the port was specified in the FANCYLOG_SERVER_PORT env var. this takes precedence - if 'FANCYLOG_SERVER_PORT' in os.environ: - port = int(os.environ['FANCYLOG_SERVER_PORT']) - port = int(port) + port = os.environ.get('FANCYLOG_SERVER_PORT', port) - logToUDP(server, port) + logToUDP(server, int(port)) _default_logTo = logToUDP else: # log to screen by default diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..047e4938ee 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -736,8 +736,7 @@ def prepare_main(args=None, logfile=None, testing=None): # if $CDPATH is set, unset it, it'll only cause trouble... # see https://github.com/easybuilders/easybuild-framework/issues/2944 - if 'CDPATH' in os.environ: - del os.environ['CDPATH'] + os.environ.pop('CDPATH', None) # When EB is run via `exec` the special bash variable $_ is not set # So emulate this here to allow (module) scripts depending on that to work diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index d68755ff24..f6055bccc1 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -83,9 +83,9 @@ def setvar(key, value, verbose=True): :param verbose: include message in dry run output for defining this environment variable """ - if key in os.environ: + try: oldval_info = "previous value: '%s'" % os.environ[key] - else: + except KeyError: oldval_info = "previously undefined" # os.putenv() is not necessary. os.environ will call this. os.environ[key] = value diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index f2d74c2137..b651502e95 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -307,9 +307,9 @@ def check_module_function(self, allow_mismatch=False, regex=None): """Check whether selected module tool matches 'module' function definition.""" if self.testing: # grab 'module' function definition from environment if it's there; only during testing - if 'module' in os.environ: + try: out, ec = os.environ['module'], 0 - else: + except KeyError: out, ec = None, 1 else: cmd = "type module" @@ -1561,9 +1561,7 @@ def get_software_root(name, with_env_var=False): """ env_var = get_software_root_env_var_name(name) - root = None - if env_var in os.environ: - root = os.getenv(env_var) + root = os.getenv(env_var) if with_env_var: res = (root, env_var) @@ -1631,9 +1629,7 @@ def get_software_version(name): """ env_var = get_software_version_env_var_name(name) - version = None - if env_var in os.environ: - version = os.getenv(env_var) + version = os.getenv(env_var) return version diff --git a/test/framework/environment.py b/test/framework/environment.py index 9a81e17486..18359fc776 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -90,8 +90,7 @@ def test_modify_env(self): # prepare test environment first: # keys in new_env should not be set yet, keys in old_env are expected to be set for key in new_env_vars: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) for key in old_env_vars: os.environ[key] = old_env_vars[key] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3b879bef85..80ee733ded 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1505,8 +1505,7 @@ def test_find_flexlm_license(self): lic_server = '1234@example.license.server' # make test robust against environment in which $LM_LICENSE_FILE is defined - if 'LM_LICENSE_FILE' in os.environ: - del os.environ['LM_LICENSE_FILE'] + os.environ.pop('LM_LICENSE_FILE', None) # default return value self.assertEqual(ft.find_flexlm_license(), ([], None)) @@ -2630,8 +2629,7 @@ def test_find_eb_script(self): """Test find_eb_script function.""" # make sure $EB_SCRIPT_PATH is not set already (used as fallback mechanism in find_eb_script) - if 'EB_SCRIPT_PATH' in os.environ: - del os.environ['EB_SCRIPT_PATH'] + os.environ.pop('EB_SCRIPT_PATH', None) self.assertExists(ft.find_eb_script('rpath_args.py')) self.assertExists(ft.find_eb_script('rpath_wrapper_template.sh.in')) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index a0c6131140..45d7402523 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -900,8 +900,7 @@ def test_getenv_cmd(self): if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: # can't have $LMOD_QUIET set when testing with Tcl syntax, # otherwise we won't get the output produced by the test module file... - if 'LMOD_QUIET' in os.environ: - del os.environ['LMOD_QUIET'] + os.environ.pop('LMOD_QUIET', None) self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) diff --git a/test/framework/modules.py b/test/framework/modules.py index 4a8a6b15df..10ae419294 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -99,8 +99,7 @@ def test_run_module(self): testdir = os.path.dirname(os.path.abspath(__file__)) for key in ['EBROOTGCC', 'EBROOTOPENMPI', 'EBROOTOPENBLAS']: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) # arguments can be passed in two ways: multiple arguments, or just 1 list argument self.modtool.run_module('load', 'GCC/6.4.0-2.28') @@ -447,8 +446,7 @@ def test_load(self): self.assertEqual(self.modtool.loaded_modules()[-1], 'GCC/6.4.0-2.28') # set things up for checking that GCC does *not* get reloaded when requested - if 'EBROOTGCC' in os.environ: - del os.environ['EBROOTGCC'] + os.environ.pop('EBROOTGCC', None) self.modtool.load(['OpenMPI/2.1.2-GCC-6.4.0-2.28']) if isinstance(self.modtool, Lmod): # order of loaded modules only changes with Lmod @@ -1025,8 +1023,7 @@ def test_modules_tool_stateless(self): init_config() # make sure $LMOD_DEFAULT_MODULEPATH, since Lmod picks it up and tweaks $MODULEPATH to match it - if 'LMOD_DEFAULT_MODULEPATH' in os.environ: - del os.environ['LMOD_DEFAULT_MODULEPATH'] + os.environ.pop('LMOD_DEFAULT_MODULEPATH', None) self.reset_modulepath([os.path.join(self.test_prefix, 'Core')]) @@ -1048,8 +1045,7 @@ def test_modules_tool_stateless(self): self.modtool.load(['OpenMPI/2.1.2']) self.modtool.purge() - if 'LMOD_DEFAULT_MODULEPATH' in os.environ: - del os.environ['LMOD_DEFAULT_MODULEPATH'] + os.environ.pop('LMOD_DEFAULT_MODULEPATH', None) # reset $MODULEPATH, obtain new ModulesTool instance, # which should not remember anything w.r.t. previous $MODULEPATH value diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 59c6872b93..894bdce009 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -200,8 +200,7 @@ def tearDown(self): if self.orig_module is not None: os.environ['module'] = self.orig_module else: - if 'module' in os.environ: - del os.environ['module'] + os.environ.pop('module', None) def suite(): diff --git a/test/framework/options.py b/test/framework/options.py index f3173dbb87..06cb14eec4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3154,8 +3154,7 @@ def test_show_default_configfiles(self): home = os.environ['HOME'] for envvar in ['XDG_CONFIG_DIRS', 'XDG_CONFIG_HOME']: - if envvar in os.environ: - del os.environ[envvar] + os.environ.pop(envvar, None) reload(easybuild.tools.options) args = [ @@ -4957,8 +4956,7 @@ def test_show_config_cfg_levels(self): """Test --show-config in relation to how configuring across multiple configuration levels interacts with it.""" # make sure default module syntax is used - if 'EASYBUILD_MODULE_SYNTAX' in os.environ: - del os.environ['EASYBUILD_MODULE_SYNTAX'] + os.environ.pop('EASYBUILD_MODULE_SYNTAX', None) # configuring --modules-tool and --module-syntax on different levels should NOT cause problems # cfr. bug report https://github.com/easybuilders/easybuild-framework/issues/2564 @@ -4984,8 +4982,7 @@ def test_modules_tool_vs_syntax_check(self): """Verify that check for modules tool vs syntax works.""" # make sure default module syntax is used - if 'EASYBUILD_MODULE_SYNTAX' in os.environ: - del os.environ['EASYBUILD_MODULE_SYNTAX'] + os.environ.pop('EASYBUILD_MODULE_SYNTAX', None) # using EnvironmentModulesC modules tool with default module syntax (Lua) is a problem os.environ['EASYBUILD_MODULES_TOOL'] = 'EnvironmentModulesC' @@ -6764,8 +6761,7 @@ def test_easystack_opts(self): mod_ext = '.lua' if get_module_syntax() == 'Lua' else '' # make sure that $EBROOTLIBTOY is not defined - if 'EBROOTLIBTOY' in os.environ: - del os.environ['EBROOTLIBTOY'] + os.environ.pop('EBROOTLIBTOY', None) # libtoy module should be installed, module file should at least set EBROOTLIBTOY mod_dir = os.path.join(self.test_installpath, 'modules', 'all') diff --git a/test/framework/robot.py b/test/framework/robot.py index 4f226bae35..89e24d008f 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -128,8 +128,7 @@ def tearDown(self): config.modules_tool = ORIG_MODULES_TOOL ecec.modules_tool = ORIG_ECEC_MODULES_TOOL if ORIG_MODULE_FUNCTION is None: - if 'module' in os.environ: - del os.environ['module'] + os.environ.pop('module', None) else: os.environ['module'] = ORIG_MODULE_FUNCTION self.modtool = self.orig_modtool diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index aacde3baf7..2d5511776d 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -181,8 +181,7 @@ def test_toolchain_prepare_sysroot(self): # clean environment self.unset_compiler_env_vars() - if 'PKG_CONFIG_PATH' in os.environ: - del os.environ['PKG_CONFIG_PATH'] + os.environ.pop('PKG_CONFIG_PATH', None) self.assertEqual(os.getenv('PKG_CONFIG_PATH'), None) @@ -227,8 +226,7 @@ def unset_compiler_env_vars(self): env_vars.extend(['OMPI_%s' % x for x in comp_env_vars]) for key in env_vars: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) def test_toolchain_compiler_env_vars(self): """Test whether environment variables for compilers are defined by toolchain mechanism.""" @@ -345,8 +343,7 @@ def test_toolchain_compiler_env_vars(self): init_config(build_options={'minimal_build_env': 'CC:gcc,CXX:g++,CFLAGS:-O2,CXXFLAGS:-O3 -g,FC:gfortan'}) for key in ['CFLAGS', 'CXXFLAGS', 'FC']: - if key in os.environ: - del os.environ[key] + os.environ.pop(key, None) self.mock_stderr(True) self.mock_stdout(True) @@ -2270,8 +2267,7 @@ def test_rpath_args_script(self): """Test rpath_args.py script""" # $LIBRARY_PATH affects result of rpath_args.py, so make sure it's not set - if 'LIBRARY_PATH' in os.environ: - del os.environ['LIBRARY_PATH'] + os.environ.pop('LIBRARY_PATH', None) script = find_eb_script('rpath_args.py') diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9ccb3c08df..faac2a7e76 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -344,8 +344,7 @@ def test_toy_tweaked(self): expected += "oh hai!" # setting $LMOD_QUIET results in suppression of printed message with Lmod & module files in Tcl syntax - if 'LMOD_QUIET' in os.environ: - del os.environ['LMOD_QUIET'] + os.environ.pop('LMOD_QUIET', None) self.modtool.use(os.path.join(self.test_installpath, 'modules', 'all')) out = self.modtool.run_module('load', 'toy/0.0-tweaked', return_output=True) diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 37a9b29318..6a9abed4a5 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -207,8 +207,7 @@ def disallow_deprecated_behaviour(self): def allow_deprecated_behaviour(self): """Restore EasyBuild version to what it was originally, to allow triggering deprecated behaviour.""" - if 'EASYBUILD_DEPRECATED' in os.environ: - del os.environ['EASYBUILD_DEPRECATED'] + os.environ.pop('EASYBUILD_DEPRECATED', None) eb_build_log.CURRENT_VERSION = self.orig_current_version @contextmanager @@ -278,8 +277,7 @@ def reset_modulepath(self, modpaths): # make very sure $MODULEPATH is totally empty # some paths may be left behind, e.g. when they contain environment variables # example: "module unuse Modules/$MODULE_VERSION/modulefiles" may not yield the desired result - if 'MODULEPATH' in os.environ: - del os.environ['MODULEPATH'] + os.environ.pop('MODULEPATH', None) for modpath in modpaths: self.modtool.add_module_path(modpath, set_mod_paths=False) self.modtool.set_mod_paths() From a748eaf2e8f65f73b0e8557162567143de09f11a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 11:11:07 +0100 Subject: [PATCH 015/171] Show empty description for license instead of "None" --- easybuild/tools/docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4fcf1ad466..1d049a19c1 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -289,7 +289,7 @@ def avail_easyconfig_licenses_md(): lics = sorted(EASYCONFIG_LICENSES_DICT.items()) table_values = [ ["``%s``" % lic().name for _, lic in lics], - [lic().description for _, lic in lics], + [lic().description or '' for _, lic in lics], ["``%s``" % lic().version for _, lic in lics], ] From 6e919e4ece21b8175d0dee9f8c0fdce8f6f1dcbc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 10:03:19 +0100 Subject: [PATCH 016/171] Use more performant and concease dict construction The pattern `dict([(key, value) for key, value in (construction)])` is very verbose and creates an intermediate list of tuples. Replace by `{key: value for key, value in (construction)}` to let Python core handle it without unncessary intermediates. --- easybuild/base/generaloption.py | 5 +++-- easybuild/framework/easyconfig/default.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 16 ++++++++-------- .../easyconfig/format/pyheaderconfigobj.py | 6 +++--- easybuild/framework/easyconfig/format/version.py | 2 +- easybuild/framework/easyconfig/templates.py | 2 +- easybuild/toolchains/mpi/craympich.py | 2 +- easybuild/toolchains/mpi/fujitsumpi.py | 2 +- easybuild/toolchains/mpi/mpich.py | 2 +- easybuild/toolchains/mpi/mpitrampoline.py | 2 +- easybuild/toolchains/mpi/openmpi.py | 2 +- easybuild/tools/config.py | 4 ++-- easybuild/tools/docs.py | 7 ++----- easybuild/tools/environment.py | 2 +- easybuild/tools/job/backend.py | 2 +- easybuild/tools/job/pbs_python.py | 2 +- easybuild/tools/module_generator.py | 2 +- .../module_naming_scheme/hierarchical_mns.py | 2 +- .../tools/module_naming_scheme/utilities.py | 2 +- easybuild/tools/modules.py | 8 ++++---- easybuild/tools/options.py | 2 +- easybuild/tools/package/utilities.py | 2 +- easybuild/tools/repository/repository.py | 2 +- test/framework/easyconfig.py | 2 +- test/framework/filetools.py | 2 +- 25 files changed, 41 insertions(+), 43 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..9bb5e32e50 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -199,7 +199,8 @@ class ExtOption(CompleterOption): ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']] - TYPE_CHECKER = dict([(x, check_str_list_tuple) for x in TYPE_STRLIST] + list(Option.TYPE_CHECKER.items())) + TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST} + TYPE_CHECKER.update(Option.TYPE_CHECKER) TYPES = tuple(TYPE_STRLIST + list(Option.TYPES)) BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG @@ -807,7 +808,7 @@ def get_env_options(self): epilogprefixtxt += "eg. --some-opt is same as setting %(prefix)s_SOME_OPT in the environment." self.epilog.append(epilogprefixtxt % {'prefix': self.envvar_prefix}) - candidates = dict([(k, v) for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)]) + candidates = {k: v for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)} for opt in self._get_all_options(): if opt._long_opts is None: diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..fb6e5fc657 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -60,7 +60,7 @@ # we use a tuple here so we can sort them based on the numbers CATEGORY_NAMES = ['BUILD', 'CUSTOM', 'DEPENDENCIES', 'EXTENSIONS', 'FILEMANAGEMENT', 'HIDDEN', 'LICENSE', 'MANDATORY', 'MODULES', 'OTHER', 'TOOLCHAIN'] -ALL_CATEGORIES = dict((name, eval(name)) for name in CATEGORY_NAMES) +ALL_CATEGORIES = {name: eval(name) for name in CATEGORY_NAMES} # List of tuples. Each tuple has the following format (key, [default, help text, category]) DEFAULT_CONFIG = { diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ff4ed3562c..9cbbd44094 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -303,7 +303,7 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): """ # obtain list of all possible subtoolchains _, all_tc_classes = search_toolchain('') - subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) + subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes} optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1) @@ -985,7 +985,7 @@ def filter_hidden_deps(self): faulty_deps = [] # obtain reference to original lists, so their elements can be changed in place - deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']]) + deps = {key: self.get_ref(key) for key in ('dependencies', 'builddependencies', 'hiddendependencies')} if 'builddependencies' in self.iterate_options: deplists = copy.deepcopy(deps['builddependencies']) @@ -1229,11 +1229,11 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals # templated values should be dumped unresolved with self.disable_templating(): # build dict of default values - default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG]) - default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options])) + default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG} + default_values.update({key: self.extra_options[key][0] for key in self.extra_options}) self.generate_template_values() - templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS]) + templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS} # create reverse map of templates, to inject template values where possible # longer template values are considered first, shorter template keys get preference over longer ones @@ -1520,7 +1520,6 @@ def _parse_dependency(self, dep, hidden=False, build_only=False): # convert tuple to string otherwise python might complain about the formatting self.log.debug("Parsing %s as a dependency" % str(dep)) - attr = ['name', 'version', 'versionsuffix', 'toolchain'] dependency = { # full/short module names 'full_mod_name': None, @@ -1576,6 +1575,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False): raise EasyBuildError("Incorrect external dependency specification: %s", dep) else: # non-external dependency: tuple (or list) that specifies name/version(/versionsuffix(/toolchain)) + attr = ('name', 'version', 'versionsuffix', 'toolchain') dependency.update(dict(zip(attr, dep))) else: @@ -2049,7 +2049,7 @@ def resolve_template(value, tmpl_dict): elif isinstance(value, tuple): value = tuple(resolve_template(list(value), tmpl_dict)) elif isinstance(value, dict): - value = dict((resolve_template(k, tmpl_dict), resolve_template(v, tmpl_dict)) for k, v in value.items()) + value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()} return value @@ -2256,7 +2256,7 @@ def verify_easyconfig_filename(path, specs, parsed_ec=None): for ec in ecs: found_fullver = det_full_ec_version(ec['ec']) if ec['ec']['name'] != specs['name'] or found_fullver != fullver: - subspec = dict((key, specs[key]) for key in ['name', 'toolchain', 'version', 'versionsuffix']) + subspec = {key: specs[key] for key in ('name', 'toolchain', 'version', 'versionsuffix')} error_msg = "Contents of %s does not match with filename" % path error_msg += "; expected filename based on contents: %s-%s.eb" % (ec['ec']['name'], found_fullver) error_msg += "; expected (relevant) parameters based on filename %s: %s" % (os.path.basename(path), subspec) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index b9fd31bf92..72454efd7b 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -51,9 +51,9 @@ def build_easyconfig_constants_dict(): """Make a dictionary with all constants that can be used""" all_consts = [ - ('TEMPLATE_CONSTANTS', dict([(x[0], x[1]) for x in TEMPLATE_CONSTANTS])), - ('EASYCONFIG_CONSTANTS', dict([(key, val[0]) for key, val in EASYCONFIG_CONSTANTS.items()])), - ('EASYCONFIG_LICENSES', dict([(klass().name, name) for name, klass in EASYCONFIG_LICENSES_DICT.items()])), + ('TEMPLATE_CONSTANTS', {x[0]: x[1] for x in TEMPLATE_CONSTANTS}), + ('EASYCONFIG_CONSTANTS', {key: val[0] for key, val in EASYCONFIG_CONSTANTS.items()}), + ('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}), ] err = [] const_dict = {} diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index fe3b1a2316..ce20ca94a3 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -69,7 +69,7 @@ class VersionOperator(object): '<': op.lt, '<=': op.le, } - REVERSE_OPERATOR_MAP = dict([(v, k) for k, v in OPERATOR_MAP.items()]) + REVERSE_OPERATOR_MAP = {v: k for k, v in OPERATOR_MAP.items()} INCLUDE_OPERATORS = ['==', '>=', '<='] # these operators *include* the (version) boundary ORDERED_OPERATORS = ['==', '>', '>=', '<', '<='] # ordering by strictness OPERATOR_FAMILIES = [['>', '>='], ['<', '<=']] # similar operators diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index aed8db6af4..bce2d139f1 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -258,7 +258,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # step 2: define *ver and *shortver templates if TEMPLATE_SOFTWARE_VERSIONS: - name_to_prefix = dict((name.lower(), pref) for name, pref in TEMPLATE_SOFTWARE_VERSIONS) + name_to_prefix = {name.lower(): pref for name, pref in TEMPLATE_SOFTWARE_VERSIONS} deps = config.get('dependencies', []) # also consider build dependencies for *ver and *shortver templates; diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index 4c62c45519..7b071ad8da 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -53,7 +53,7 @@ class CrayMPICH(Mpi): MPI_COMPILER_MPIFC = CrayPECompiler.COMPILER_FC # no MPI wrappers, so no need to specify serial compiler - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} def _set_mpi_compiler_variables(self): """Set the MPI compiler variables""" diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py index f81e1d66f0..704eab1742 100644 --- a/easybuild/toolchains/mpi/fujitsumpi.py +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -45,7 +45,7 @@ class FujitsuMPI(Mpi): MPI_TYPE = TC_CONSTANT_MPI_TYPE_OPENMPI # OpenMPI reads from CC etc env variables - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} MPI_LINK_INFO_OPTION = '-showme:link' diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index b0c42e57d8..45b8580799 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -56,7 +56,7 @@ class Mpich(Mpi): MPI_COMPILER_MPIFC = None # clear MPI wrapper command options - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} def _set_mpi_compiler_variables(self): """Set the MPICH_{CC, CXX, F77, F90, FC} variables.""" diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 5af054661c..f3b8a332ac 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -53,7 +53,7 @@ class MPItrampoline(Mpi): MPI_COMPILER_MPIFC = None # MPItrampoline reads from CC etc env variables - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} MPI_LINK_INFO_OPTION = '-showme:link' diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index 630262f48b..787fefd429 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -61,7 +61,7 @@ class OpenMPI(Mpi): MPI_COMPILER_MPIFC = None # OpenMPI reads from CC etc env variables - MPI_SHARED_OPTION_MAP = dict([('_opt_%s' % var, '') for var, _ in MPI_COMPILER_VARIABLES]) + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} MPI_LINK_INFO_OPTION = '-showme:link' diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ee8eb5fc53..09d18dbaf3 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -574,7 +574,7 @@ def init_build_options(build_options=None, cmdline_options=None): cmdline_options.accept_eula_for = cmdline_options.accept_eula cmdline_build_option_names = [k for ks in BUILD_OPTIONS_CMDLINE.values() for k in ks] - active_build_options.update(dict([(key, getattr(cmdline_options, key)) for key in cmdline_build_option_names])) + active_build_options.update({key: getattr(cmdline_options, key) for key in cmdline_build_option_names}) # other options which can be derived but have no perfectly matching cmdline option active_build_options.update({ 'check_osdeps': not cmdline_options.ignore_osdeps, @@ -597,7 +597,7 @@ def init_build_options(build_options=None, cmdline_options=None): for opt in build_options_by_default[default]: bo[opt] = [] else: - bo.update(dict([(opt, default) for opt in build_options_by_default[default]])) + bo.update({opt: default for opt in build_options_by_default[default]}) bo.update(active_build_options) # BuildOptions is a singleton, so any future calls to BuildOptions will yield the same instance diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..57671a8b99 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1028,12 +1028,9 @@ def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') - # filter deprecated 'dummy' toolchain - all_tcs = [x for x in all_tcs if x.NAME != DUMMY_TOOLCHAIN_NAME] - all_tcs_names = [x.NAME for x in all_tcs] - # start with dict that maps toolchain name to corresponding subclass of Toolchain - tcs = dict(zip(all_tcs_names, all_tcs)) + # filter deprecated 'dummy' toolchain + tcs = {tc.NAME: tc for tc in all_tcs if tc.NAME != DUMMY_TOOLCHAIN_NAME} for tcname in sorted(tcs): tcc = tcs[tcname] diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index d68755ff24..115a08891d 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -136,7 +136,7 @@ def read_environment(env_vars, strict=False): :param env_vars: a dict with key a name, value a environment variable name :param strict: boolean, if True enforces that all specified environment variables are found """ - result = dict([(k, os.environ.get(v)) for k, v in env_vars.items() if v in os.environ]) + result = {k: os.environ.get(v) for k, v in env_vars.items() if v in os.environ} if not len(env_vars) == len(result): missing = ','.join(["%s / %s" % (k, v) for k, v in env_vars.items() if k not in result]) diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index c92ead6a58..0b1f69eb08 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -104,7 +104,7 @@ def avail_job_backends(check_usable=True): Return all known job execution backends. """ import_available_modules('easybuild.tools.job') - class_dict = dict([(x.__name__, x) for x in get_subclasses(JobBackend)]) + class_dict = {x.__name__: x for x in get_subclasses(JobBackend)} return class_dict diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index a4a020988e..1ad13738ba 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -484,7 +484,7 @@ def info(self, types=None): # only expect to have a list with one element j = jobs[0] # convert attribs into useable dict - job_details = dict([(attrib.name, attrib.value) for attrib in j.attribs]) + job_details = {attrib.name: attrib.value for attrib in j.attribs} # manually set 'id' attribute job_details['id'] = j.name self.log.debug("Found jobinfo %s" % job_details) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0c19b93c8b..723753e31a 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -59,7 +59,7 @@ def avail_module_generators(): """ Return all known module syntaxes. """ - return dict([(k.SYNTAX, k) for k in get_subclasses(ModuleGenerator)]) + return {k.SYNTAX: k for k in get_subclasses(ModuleGenerator)} def module_generator(app, fake=False): diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 9f2e026eb8..0de0eb5107 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -120,7 +120,7 @@ def det_toolchain_compilers_name_version(self, tc_comps): res = None else: if len(tc_comps) > 0 and tc_comps[0]: - comp_versions = dict([(comp['name'], self.det_full_version(comp)) for comp in tc_comps]) + comp_versions = {comp['name']: self.det_full_version(comp) for comp in tc_comps} comp_names = comp_versions.keys() key = ','.join(sorted(comp_names)) if key in COMP_NAME_VERSION_TEMPLATES: diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index e46609d22b..2878cbf1a1 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -86,7 +86,7 @@ def avail_module_naming_schemes(): import_available_modules('easybuild.tools.module_naming_scheme') # construct name-to-class dict of available module naming scheme - avail_mnss = dict([(x.__name__, x) for x in get_subclasses(ModuleNamingScheme)]) + avail_mnss = {x.__name__: x for x in get_subclasses(ModuleNamingScheme)} return avail_mnss diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index f2d74c2137..a6a81781dc 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -851,7 +851,7 @@ def run_module(self, *args, **kwargs): # this needs to be taken into account when updating the environment via produced output, see below # keep track of current values of select env vars, so we can correct the adjusted values below - prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS]) + prev_ld_values = {key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS} # Change the environment try: @@ -1126,7 +1126,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, if modpath_exts is None: # only retain dependencies that have a non-empty lists of $MODULEPATH extensions - modpath_exts = dict([(k, v) for k, v in self.modpath_extensions_for(deps).items() if v]) + modpath_exts = {k: v for k, v in self.modpath_extensions_for(deps).items() if v} self.log.debug("Non-empty lists of module path extensions for dependencies: %s" % modpath_exts) mods_to_top = [] @@ -1157,7 +1157,7 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, path = mods_to_top[:] if mods_to_top: # remove retained dependencies from the list, since we're climbing up the module tree - remaining_modpath_exts = dict([m for m in modpath_exts.items() if not m[0] in mods_to_top]) + remaining_modpath_exts = {m: v for m, v in modpath_exts.items() if m not in mods_to_top} self.log.debug("Path to top from %s extended to %s, so recursing to find way to the top", mod_name, mods_to_top) @@ -1662,7 +1662,7 @@ def avail_modules_tools(): """ Return all known modules tools. """ - class_dict = dict([(x.__name__, x) for x in get_subclasses(ModulesTool)]) + class_dict = {x.__name__: x for x in get_subclasses(ModulesTool)} # filter out legacy Modules class if 'Modules' in class_dict: del class_dict['Modules'] diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b840bc4d53..3364256957 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -244,7 +244,7 @@ def __init__(self, *args, **kwargs): # update or define go_configfiles_initenv in named arguments to pass to parent constructor go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) for section, constants in self.go_cfg_constants.items(): - constants = dict([(name, value) for (name, (value, _)) in constants.items()]) + constants = {name: value for (name, (value, _)) in constants.items()} go_cfg_initenv.setdefault(section, {}).update(constants) super(EasyBuildOptions, self).__init__(*args, **kwargs) diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 087084c687..8de28e5e5b 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -58,7 +58,7 @@ def avail_package_naming_schemes(): They are loaded from the easybuild.package.package_naming_scheme namespace """ import_available_modules('easybuild.tools.package.package_naming_scheme') - class_dict = dict([(x.__name__, x) for x in get_subclasses(PackageNamingScheme)]) + class_dict = {x.__name__: x for x in get_subclasses(PackageNamingScheme)} return class_dict diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index cbd33ab654..effa5ae0bf 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -146,7 +146,7 @@ def avail_repositories(check_useable=True): """ import_available_modules('easybuild.tools.repository') - class_dict = dict([(x.__name__, x) for x in get_subclasses(Repository) if x.USABLE or not check_useable]) + class_dict = {x.__name__: x for x in get_subclasses(Repository) if x.USABLE or not check_useable} if 'FileRepository' not in class_dict: raise EasyBuildError("avail_repositories: FileRepository missing from list of repositories") diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index dda5c0a05b..219b481b69 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3621,7 +3621,7 @@ def test_resolve_template(self): def test_det_subtoolchain_version(self): """Test det_subtoolchain_version function""" _, all_tc_classes = search_toolchain('') - subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) + subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes} optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)) current_tc = {'name': 'fosscuda', 'version': '2018a'} diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3b879bef85..cf28cbc257 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -320,7 +320,7 @@ def test_checksums(self): self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum) # make sure faulty checksums are reported - broken_checksums = dict([(typ, val[:-3] + 'foo') for (typ, val) in known_checksums.items()]) + broken_checksums = {typ: (val[:-3] + 'foo') for (typ, val) in known_checksums.items()} for checksum_type, checksum in broken_checksums.items(): self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum) self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum))) From 13191968a1b43436c5df63d8cd52edc8d503560c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 13:41:47 +0100 Subject: [PATCH 017/171] Remove typo Co-authored-by: Kenneth Hoste --- .../tools/module_naming_scheme/test_module_naming_scheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 02934e5745..5d8dec26c8 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -64,6 +64,6 @@ def det_module_symlink_paths(self, ec): def is_short_modname_for(self, modname, name): """ - Determine whether the specified (short) module name is a moduleq for software with the specified name. + Determine whether the specified (short) module name is a module for software with the specified name. """ return modname.find(name) != -1 From cd1c1f5abc52bdfa6b9618fa8980f0f184a19e93 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 11:08:04 +0100 Subject: [PATCH 018/171] Fix compatibility with Python2 --- easybuild/tools/modules.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a6a81781dc..8674c4350b 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -851,7 +851,9 @@ def run_module(self, *args, **kwargs): # this needs to be taken into account when updating the environment via produced output, see below # keep track of current values of select env vars, so we can correct the adjusted values below - prev_ld_values = {key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS} + # Identical to `{key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS}` + # but Python 2 treats that as a local function and refused the `exec` below + prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS]) # Change the environment try: From cf02d8333a67900f7d04e96522232d8bef5760ed Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 13:40:56 +0100 Subject: [PATCH 019/171] Remove parens from dict construction for consistency --- easybuild/tools/options.py | 2 +- test/framework/filetools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 3364256957..75fbf65b9b 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -244,7 +244,7 @@ def __init__(self, *args, **kwargs): # update or define go_configfiles_initenv in named arguments to pass to parent constructor go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) for section, constants in self.go_cfg_constants.items(): - constants = {name: value for (name, (value, _)) in constants.items()} + constants = {name: value for name, (value, _) in constants.items()} go_cfg_initenv.setdefault(section, {}).update(constants) super(EasyBuildOptions, self).__init__(*args, **kwargs) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index cf28cbc257..d15d0df7c4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -320,7 +320,7 @@ def test_checksums(self): self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum) # make sure faulty checksums are reported - broken_checksums = {typ: (val[:-3] + 'foo') for (typ, val) in known_checksums.items()} + broken_checksums = {typ: (val[:-3] + 'foo') for typ, val in known_checksums.items()} for checksum_type, checksum in broken_checksums.items(): self.assertFalse(ft.compute_checksum(fp, checksum_type=checksum_type) == checksum) self.assertFalse(ft.verify_checksum(fp, (checksum_type, checksum))) From 3830181bb4a6fad4f8d3f71f884961345fc47eae Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 16:13:31 +0100 Subject: [PATCH 020/171] Cleanup uses of `getattr` and `hasattr` `hasattr` is basically calling `getattr` and checking for `AttributeError`. It also supports a default value in case the attribute is not found. Make use of that default value and catch the exception directly instead of checking `hasattr` where that makes sense. --- easybuild/base/fancylogger.py | 2 +- easybuild/base/generaloption.py | 2 +- easybuild/base/optcomplete.py | 21 ++++++++-------- .../easyconfig/format/pyheaderconfigobj.py | 11 +++++---- easybuild/tools/configobj.py | 2 +- easybuild/tools/docs.py | 7 +++--- easybuild/tools/github.py | 4 ++-- easybuild/tools/systemtools.py | 4 ++-- easybuild/tools/toolchain/linalg.py | 10 ++++---- easybuild/tools/toolchain/toolchain.py | 24 +++++++++++-------- easybuild/tools/toolchain/utilities.py | 11 +++++---- test/framework/systemtools.py | 7 ++---- 12 files changed, 53 insertions(+), 52 deletions(-) diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index b5a63477c6..93582b0cf1 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -286,7 +286,7 @@ def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=Non overwrite make record to use a fancy record (with more options) """ logrecordcls = logging.LogRecord - if hasattr(self, 'fancyrecord') and self.fancyrecord: + if getattr(self, 'fancyrecord', None): logrecordcls = FancyLogRecord try: new_msg = str(msg) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 51f7daafe3..49b1168008 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1031,7 +1031,7 @@ def main_options(self): # make_init is deprecated if hasattr(self, 'make_init'): self.log.debug('main_options: make_init is deprecated. Rename function to main_options.') - getattr(self, 'make_init')() + self.make_init() else: # function names which end with _options and do not start with main or _ reg_main_options = re.compile("^(?!_|main).*_options$") diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 9a2bc8a127..3fa75a6635 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -513,14 +513,15 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete if option: if option.nargs > 0: optarg = True - if hasattr(option, 'completer'): + try: completer = option.completer - elif option.choices: - completer = ListCompleter(option.choices) - elif option.type in ('string',): - completer = opt_completer - else: - completer = NoneCompleter() + except AttributeError: + if option.choices: + completer = ListCompleter(option.choices) + elif option.type in ('string',): + completer = opt_completer + else: + completer = NoneCompleter() # Warn user at least, it could help him figure out the problem. elif hasattr(option, 'completer'): msg = "Error: optparse option with a completer does not take arguments: %s" % (option) @@ -616,11 +617,9 @@ class CmdComplete(object): def autocomplete(self, completer=None): parser = OPTIONPARSER_CLASS(self.__doc__.strip()) if hasattr(self, 'addopts'): - fnc = getattr(self, 'addopts') - fnc(parser) + self.addopts(parser) - if hasattr(self, 'completer'): - completer = getattr(self, 'completer') + completer = getattr(self, 'completer', completer) return autocomplete(parser, completer) diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index b9fd31bf92..37e14b529e 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -233,12 +233,13 @@ def pyheader_env(self): current_builtins = globals()['__builtins__'] builtins = {} for name in self.PYHEADER_ALLOWED_BUILTINS: - if hasattr(current_builtins, name): + try: builtins[name] = getattr(current_builtins, name) - elif isinstance(current_builtins, dict) and name in current_builtins: - builtins[name] = current_builtins[name] - else: - self.log.warning('No builtin %s found.' % name) + except AttributeError: + if isinstance(current_builtins, dict) and name in current_builtins: + builtins[name] = current_builtins[name] + else: + self.log.warning('No builtin %s found.' % name) global_vars['__builtins__'] = builtins self.log.debug("Available builtins: %s" % global_vars['__builtins__']) diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index 48c7dd348d..6ac667b5bd 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -1254,7 +1254,7 @@ def set_section(in_section, this_section): self.configspec = None return - elif getattr(infile, 'read', MISSING) is not MISSING: + elif hasattr(infile, 'read'): # This supports file like objects infile = infile.read() or [] # needs splitting into lines - but needs doing *after* decoding diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..dc707466ad 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1182,10 +1182,9 @@ def avail_toolchain_opts(name, output_format=FORMAT_TXT): tc_dict = {} for cst in ['COMPILER_SHARED_OPTS', 'COMPILER_UNIQUE_OPTS', 'MPI_SHARED_OPTS', 'MPI_UNIQUE_OPTS']: - if hasattr(tc, cst): - opts = getattr(tc, cst) - if opts is not None: - tc_dict.update(opts) + opts = getattr(tc, cst, None) + if opts is not None: + tc_dict.update(opts) return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict]) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 983dc977c4..ca013743d7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -719,9 +719,9 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin) # git checkout -b ; git pull - if hasattr(origin.refs, branch_name): + try: origin_branch = getattr(origin.refs, branch_name) - else: + except AttributeError: raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url) _log.debug("Checking out branch '%s' from remote %s", branch_name, github_url) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 78ba312203..59b8728454 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1340,8 +1340,8 @@ def det_pypkg_version(pkg_name, imported_pkg, import_name=None): except pkg_resources.DistributionNotFound as err: _log.debug("%s Python package not found: %s", pkg_name, err) - if version is None and hasattr(imported_pkg, '__version__'): - version = imported_pkg.__version__ + if version is None: + version = getattr(imported_pkg, '__version__', None) return version diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index b7ce694ff7..907993571c 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -212,9 +212,9 @@ def _set_blacs_variables(self): """Set BLACS related variables""" lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) # BLACS @@ -254,11 +254,11 @@ def _set_scalapack_variables(self): raise EasyBuildError("_set_blas_variables: SCALAPACK_LIB not set") lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) - if hasattr(self, 'SCALAPACK_LIB_MAP') and self.SCALAPACK_LIB_MAP is not None: + if getattr(self, 'SCALAPACK_LIB_MAP', None) is not None: lib_map.update(self.SCALAPACK_LIB_MAP) self.SCALAPACK_LIB = self.variables.nappend('LIBSCALAPACK_ONLY', [x % lib_map for x in self.SCALAPACK_LIB]) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a16fe35dc0..4c30faf19a 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -163,13 +163,13 @@ def _is_toolchain_for(cls, name): """see if this class can provide support for toolchain named name""" # TODO report later in the initialization the found version if name: - if hasattr(cls, 'NAME') and name == cls.NAME: - return True - else: + try: + return name == cls.NAME + except AttributeError: return False else: # is no name is supplied, check whether class can be used as a toolchain - return hasattr(cls, 'NAME') and cls.NAME + return bool(getattr(cls, 'NAME', None)) _is_toolchain_for = classmethod(_is_toolchain_for) @@ -330,10 +330,12 @@ def _copy_class_constants(self): if key not in self.CLASS_CONSTANT_COPIES: self.CLASS_CONSTANT_COPIES[key] = {} for cst in self.CLASS_CONSTANTS_TO_RESTORE: - if hasattr(self, cst): - self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) - else: + try: + value = getattr(self, cst) + except AttributeError: raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self) + else: + self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(value) self.log.devel("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) @@ -342,10 +344,12 @@ def _restore_class_constants(self): key = self.__class__ for cst in self.CLASS_CONSTANT_COPIES[key]: newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst]) - if hasattr(self, cst): - self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst)) - else: + try: + oldval = getattr(self, cst) + except AttributeError: self.log.devel("Restoring (currently undefined) class constant '%s' to %s", cst, newval) + else: + self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, oldval) setattr(self, cst, newval) diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 48a586c567..56242b8fe4 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -63,7 +63,7 @@ def search_toolchain(name): package = easybuild.tools.toolchain check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX - if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name): + if not getattr(package, check_attr_name, None): # import all available toolchains, so we know about them tc_modules = import_available_modules('easybuild.toolchains') @@ -76,7 +76,7 @@ def search_toolchain(name): if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: - elem_name = elem.__name__ if hasattr(elem, '__name__') else elem + elem_name = getattr(elem, '__name__', elem) _log.debug("Adding %s to list of imported classes used for looking for constants", elem_name) mod_classes.append(elem) @@ -89,13 +89,14 @@ def search_toolchain(name): tc_const_value = getattr(mod_class_mod, elem) _log.debug("Found constant %s ('%s') in module %s, adding it to %s", tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) - if hasattr(package, tc_const_name): + try: cur_value = getattr(package, tc_const_name) + except AttributeError: + setattr(package, tc_const_name, tc_const_value) + else: if not tc_const_value == cur_value: raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", package.__name__, tc_const_name, cur_value, tc_const_value) - else: - setattr(package, tc_const_name, tc_const_value) # indicate that processing of toolchain constants is done, so it's not done again setattr(package, check_attr_name, True) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 1707d6ec54..45f51991ff 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -366,17 +366,14 @@ def setUp(self): self.orig_is_readable = st.is_readable self.orig_read_file = st.read_file self.orig_run_cmd = st.run_cmd - self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None + self.orig_platform_dist = getattr(st.platform, 'dist', None) self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version self.orig_sys_version_info = st.sys.version_info self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC self.orig_HAVE_DISTRO = st.HAVE_DISTRO self.orig_ETC_OS_RELEASE = st.ETC_OS_RELEASE - if hasattr(st, 'archspec_cpu_host'): - self.orig_archspec_cpu_host = st.archspec_cpu_host - else: - self.orig_archspec_cpu_host = None + self.orig_archspec_cpu_host = getattr(st, 'archspec_cpu_host', None) def tearDown(self): """Cleanup after systemtools test.""" From 1fad79e11d714a66acd7afa747d461e48ce60399 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Mon, 30 Oct 2023 20:52:30 +0100 Subject: [PATCH 021/171] Adapt VERSION_REGEXP for EnvironmentModules When using an Environment Modules version built from git repository, version number contains git branch name, number of commit since last released version and commit hash. This commit adapts VERSION_REGEXP for EnvironmentModules class to allow using development or locally adapted versions of Environment Modules with EasyBuild. Fixes #4126 --- easybuild/tools/modules.py | 2 +- test/framework/modulestool.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d7181fe30..bdac20c274 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1326,7 +1326,7 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' MAX_VERSION = None - VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' def __init__(self, *args, **kwargs): """Constructor, set Environment Modules-specific class variable values.""" diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index fb991ad797..f43a91e3b3 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -209,6 +209,21 @@ def test_environment_modules_specific(self): mt = EnvironmentModules(testing=True) self.assertIsInstance(mt.loaded_modules(), list) # dummy usage + # initialize Environment Modules tool with non-official version number + # pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD + fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl') + fake_modulecmd_txt = '\n'.join([ + 'puts stderr {Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)}', + "puts {os.environ['FOO'] = 'foo'}", + ]) + write_file(fake_path, fake_modulecmd_txt) + os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR) + os.environ['_module_raw'] = "() { eval `%s' bash $*`;\n}" % fake_path + os.environ['MODULES_CMD'] = fake_path + EnvironmentModules.COMMAND = fake_path + mt = EnvironmentModules(testing=True) + self.assertTrue(os.path.samefile(mt.cmd, fake_path), "%s - %s" % (mt.cmd, fake_path)) + def tearDown(self): """Testcase cleanup.""" super(ModulesToolTest, self).tearDown() From a3cb2ccdf6a8ead993d553e00a637ad747d1c5ff Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 13:23:23 +0100 Subject: [PATCH 022/171] Get available hidden modules on EnvironmentModules Environment Modules option "--all" was introduced in version 4.6.0 to obtain hidden modules among "module avail" results. Update EnvironmentModules class to make use of this option to return hidden modules on "available" function like done on Lmod class. Update "test_avail" unit test to match new results obtain with Environment Modules 4.6+ --- easybuild/tools/modules.py | 21 +++++++++++++++++++-- test/framework/modules.py | 5 +++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 7d7181fe30..d3c6019aba 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1283,14 +1283,14 @@ def tweak_stdout(txt): return super(EnvironmentModulesTcl, self).run_module(*args, **kwargs) - def available(self, mod_name=None): + def available(self, mod_name=None, extra_args=None): """ Return a list of available modules for the given (partial) module name; use None to obtain a list of all available modules. :param mod_name: a (partial) module name for filtering (default: None) """ - mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name) + mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name, extra_args=extra_args) # strip off slash at beginning, if it's there # under certain circumstances, 'modulecmd.tcl avail' (DEISA variant) spits out available modules like this clean_mods = [mod.lstrip(os.path.sep) for mod in mods] @@ -1328,6 +1328,8 @@ class EnvironmentModules(EnvironmentModulesTcl): MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + SHOW_HIDDEN_OPTION = '--all' + def __init__(self, *args, **kwargs): """Constructor, set Environment Modules-specific class variable values.""" # ensure in-depth modulepath search (MODULES_AVAIL_INDEPTH has been introduced in v4.3) @@ -1383,6 +1385,21 @@ def check_module_output(self, cmd, stdout, stderr): else: self.log.debug("No errors detected when running module command '%s'", cmd) + def available(self, mod_name=None, extra_args=None): + """ + Return a list of available modules for the given (partial) module name; + use None to obtain a list of all available modules. + + :param mod_name: a (partial) module name for filtering (default: None) + """ + if extra_args is None: + extra_args = [] + # make hidden modules visible (requires Environment Modules 4.6.0) + if StrictVersion(self.version) >= StrictVersion('4.6.0'): + extra_args.append(self.SHOW_HIDDEN_OPTION) + + return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args) + def get_setenv_value_from_modulefile(self, mod_name, var_name): """ Get value for specific 'setenv' statement from module file for the specified module. diff --git a/test/framework/modules.py b/test/framework/modules.py index c2460c7a11..9151527aac 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -218,6 +218,11 @@ def test_avail(self): self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) + elif (isinstance(self.modtool, EnvironmentModules) + and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) + self.assertIn('toy/.0.0-deps', ms) + self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) else: self.assertEqual(len(ms), TEST_MODULES_COUNT) From 2d5db91d92c97ae3ee7ed22a50cb20f3eb56e0c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 09:04:59 +0100 Subject: [PATCH 023/171] add comment to explain why hidden bzip2 module is not included in "avail" output for Tcl-based environment modules tool --- test/framework/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index 9151527aac..2a05395b7c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -220,6 +220,7 @@ def test_avail(self): self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) From b05b07ddddfb43861015ab2eb685d73260ed8272 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:16:58 +0000 Subject: [PATCH 024/171] bump minimum required Lmod to 8.0.0 --- easybuild/tools/modules.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 36368fe6e0..7e106ae1d2 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1406,9 +1406,8 @@ class Lmod(ModulesTool): NAME = "Lmod" COMMAND = 'lmod' COMMAND_ENVIRONMENT = 'LMOD_CMD' - REQ_VERSION = '6.5.1' - DEPR_VERSION = '7.0.0' - REQ_VERSION_DEPENDS_ON = '7.6.1' + REQ_VERSION = '8.0.0' + DEPR_VERSION = '8.0.0' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" SHOW_HIDDEN_OPTION = '--show-hidden' @@ -1427,7 +1426,7 @@ def __init__(self, *args, **kwargs): super(Lmod, self).__init__(*args, **kwargs) version = StrictVersion(self.version) - self.supports_depends_on = version >= self.REQ_VERSION_DEPENDS_ON + self.supports_depends_on = True # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html if version >= '8.7.12': self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') @@ -1586,13 +1585,9 @@ def module_wrapper_exists(self, mod_name): Determine whether a module wrapper with specified name exists. First check for wrapper defined in .modulerc.lua, fall back to also checking .modulerc (Tcl syntax). """ - res = None - - # first consider .modulerc.lua with Lmod 7.8 (or newer) - if StrictVersion(self.version) >= StrictVersion('7.8'): - mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' - res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', - mod_wrapper_regex_template=mod_wrapper_regex_template) + mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' + res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', + mod_wrapper_regex_template=mod_wrapper_regex_template) # fall back to checking for .modulerc in Tcl syntax if res is None: From 21f1d5862166074dcd05d76a21f1b06c34f7022e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 15:20:24 +0100 Subject: [PATCH 025/171] fix log.deprecated statements for renamed methods for installing extensions --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/extension.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 914e8ad4a0..66a67e7b32 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1854,7 +1854,7 @@ def install_extensions(self, *args, **kwargs): """[DEPRECATED] Install extensions.""" self.log.deprecated( "Easyblock.install_extensions() is deprecated, use Easyblock.install_all_extensions() instead.", - '5.0', + '6.0', ) self.install_all_extensions(*args, **kwargs) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 31cfb8d373..1aa53656e7 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -177,7 +177,7 @@ def prerun(self): """ self.log.deprecated( "Extension.prerun() is deprecated, use Extension.pre_install_extension() instead.", - '5.0', + '6.0', ) self.pre_install_extension() @@ -193,7 +193,7 @@ def run(self, *args, **kwargs): """ self.log.deprecated( "Extension.run() is deprecated, use Extension.install_extension() instead.", - '5.0', + '6.0', ) self.install_extension(*args, **kwargs) @@ -209,7 +209,7 @@ def run_async(self, *args, **kwargs): """ self.log.deprecated( "Extension.run_async() is deprecated, use Extension.install_extension_async() instead.", - '5.0', + '6.0', ) self.install_extension_async(*args, **kwargs) @@ -225,7 +225,7 @@ def postrun(self): """ self.log.deprecated( "Extension.postrun() is deprecated, use Extension.post_install_extension() instead.", - '5.0', + '6.0', ) self.post_install_extension() From 5cdea4615623479f68d2f34169a971c5cef26e25 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:35:39 +0000 Subject: [PATCH 026/171] remove Lmod 7 testing --- .github/workflows/unit_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 08408aea75..225b3f85c6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -13,7 +13,6 @@ jobs: setup: runs-on: ubuntu-20.04 outputs: - lmod7: Lmod-7.8.22 lmod8: Lmod-8.7.6 modulesTcl: modules-tcl-1.147 modules3: modules-3.2.10 @@ -29,7 +28,6 @@ jobs: modules_tool: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - - ${{needs.setup.outputs.lmod7}} - ${{needs.setup.outputs.lmod8}} - ${{needs.setup.outputs.modulesTcl}} - ${{needs.setup.outputs.modules3}} From ed8ac7acd85c124caf20484ee2a98e3969a8c333 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 10 Jan 2024 09:44:53 +0100 Subject: [PATCH 027/171] add params modextrapaths_append and allow_append_abs_path --- easybuild/framework/easyblock.py | 8 ++++++++ easybuild/framework/easyconfig/default.py | 2 ++ easybuild/framework/easyconfig/format/format.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9f3233c265..063b2c01cc 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1421,6 +1421,14 @@ def make_module_extra(self, altroot=None, altversion=None): value, type(value)) lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path'])) + for (key, value) in self.cfg['modextrapaths_append'].items(): + if isinstance(value, string_type): + value = [value] + elif not isinstance(value, (tuple, list)): + raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple", + value, type(value)) + lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path'])) + modloadmsg = self.cfg['modloadmsg'] if modloadmsg: # add trailing newline to prevent that shell prompt is 'glued' to module load message diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dd91229d1e..dae27aca65 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -189,10 +189,12 @@ 'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS], # MODULES easyconfig parameters + 'allow_append_abs_path': [False, "Allow specifying absolute paths to append in modextrapaths_append", MODULES], 'allow_prepend_abs_path': [False, "Allow specifying absolute paths to prepend in modextrapaths", MODULES], 'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES], 'modaliases': [{}, "Aliases to be defined in module file", MODULES], 'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES], + 'modextrapaths_append': [{}, "Extra paths to be appended in module file", MODULES], 'modextravars': [{}, "Extra environment variables to be added to module file", MODULES], 'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES], 'modunloadmsg': [{}, "Message that should be printed when generated module is unloaded", MODULES], diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 9a1626c60b..d503b6703a 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -73,7 +73,7 @@ ] LAST_PARAMS = ['exts_default_options', 'exts_list', 'sanity_check_paths', 'sanity_check_commands', - 'modextrapaths', 'modextravars', + 'modextrapaths', 'modextrapaths_append', 'modextravars', 'moduleclass'] SANITY_CHECK_PATHS_DIRS = 'dirs' From 5b0d6a302501088c700fd6255a1a42bf0b7f40d4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 11 Jan 2024 13:24:49 +0100 Subject: [PATCH 028/171] Improve --check-github output Fix misdetecting an empty username as valid. Show more information on why/what failed, e.g. for creating gists or not/partially populated --git_working_dirs_path --- easybuild/tools/github.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ca013743d7..8100c8613b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2008,17 +2008,18 @@ def check_github(): github_user = build_option('github_user') github_account = build_option('github_org') or build_option('github_user') - if github_user is None: - check_res = "(none available) => FAIL" - status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False - else: + if github_user: check_res = "%s => OK" % github_user + else: + check_res = "%s => FAIL" % ('(none available)' if github_user is None else '(empty)') + status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check GitHub token print_msg("* GitHub token...", log=_log, prefix=False, newline=False) github_token = fetch_github_token(github_user) + github_token_valid = False if github_token is None: check_res = "(no token found) => FAIL" else: @@ -2027,6 +2028,7 @@ def check_github(): token_descr = partial_token + " (len: %d)" % len(github_token) if validate_github_token(github_token, github_user): check_res = "%s => OK (validated)" % token_descr + github_token_valid = True else: check_res = "%s => FAIL (validation failed)" % token_descr @@ -2119,7 +2121,7 @@ def check_github(): try: getattr(git_repo.remotes, remote_name).push(branch_name, delete=True) except GitCommandError as err: - sys.stderr.write("WARNING: failed to delete test branch from GitHub: %s\n" % err) + print_warning("failed to delete test branch from GitHub: %s" % err, log=_log) # test creating a gist print_msg("* creating gists...", log=_log, prefix=False, newline=False) @@ -2137,17 +2139,33 @@ def check_github(): if gist_url and re.match('https://gist.github.com/%s/[0-9a-f]+$' % github_user, gist_url): check_res = "OK" - else: + elif not github_user: + check_res = "FAIL (no GitHub user specified)" + elif not github_token: + check_res = "FAIL (missing github token)" + elif not github_token_valid: + check_res = "FAIL (invalid github token)" + elif gist_url: check_res = "FAIL (gist_url: %s)" % gist_url - status['--upload-test-report'] = False + else: + check_res = "FAIL" + if 'FAIL' in check_res: + status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check whether location to local working directories for Git repositories is available (not strictly needed) print_msg("* location to Git working dirs... ", log=_log, prefix=False, newline=False) git_working_dirs_path = build_option('git_working_dirs_path') if git_working_dirs_path: - check_res = "OK (%s)" % git_working_dirs_path + repos = [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO] + missing_repos = [repo for repo in repos if not os.path.exists(os.path.join(git_working_dirs_path, repo))] + if not missing_repos: + check_res = "OK (%s)" % git_working_dirs_path + elif missing_repos != repos: + check_res = "OK (%s) but missing %s (suboptimal)" % (git_working_dirs_path, ', '.join(missing_repos)) + else: + check_res = "set (%s) but not populated (suboptimal)" % git_working_dirs_path else: check_res = "not found (suboptimal)" From 8127a99c775061f85858fa4b3646cd810df33b00 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 10:03:47 +0100 Subject: [PATCH 029/171] add tests --- test/framework/easyblock.py | 19 +++++++++++++++++++ test/framework/easyconfig.py | 2 ++ test/framework/toy_build.py | 9 +++++++++ 3 files changed, 30 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 5e2407575d..df40f42d98 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -645,6 +645,7 @@ def test_make_module_extra(self): # also check how absolute paths specified in modexself.contents = '\n'.join([ self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -657,6 +658,7 @@ def test_make_module_extra(self): # allow use of absolute paths, and verify contents of module self.contents += "\nallow_prepend_abs_path = True" + self.contents += "\nallow_append_abs_path = True" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -675,6 +677,9 @@ def test_make_module_extra(self): r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path", r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar", + r"^append[-_]path.*TEST_PATH_VAR.*root.*foo", + r"^append[-_]path.*TEST_PATH_VAR.*/test/absolute/path", + r"^append[-_]path.*TEST_PATH_VAR.*root.*bar", ] for pattern in patterns: self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt)) @@ -1173,6 +1178,7 @@ def test_make_module_step(self): 'PATH': ('xbin', 'pibin'), 'CPATH': 'pi/include', } + modextrapaths_append = modextrapaths.copy() self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, @@ -1186,6 +1192,7 @@ def test_make_module_step(self): "hiddendependencies = [('test', '1.2.3'), ('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]", "modextravars = %s" % str(modextravars), "modextrapaths = %s" % str(modextrapaths), + "modextrapaths_append = %s" % str(modextrapaths_append), ]) # test if module is generated correctly @@ -1256,6 +1263,18 @@ def test_make_module_step(self): num_prepends = len(regex.findall(txt)) self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) + for (key, vals) in modextrapaths_append.items(): + if isinstance(vals, string_type): + vals = [vals] + for val in vals: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^append-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.fail("Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (name, ver) in [('GCC', '6.4.0-2.28')]: if get_module_syntax() == 'Tcl': regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index dda5c0a05b..905cf6c15b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1151,6 +1151,7 @@ def test_templating_constants(self): 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), 'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}', + 'modextrapaths_append = {"PATH_APPEND": "appended_path"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", ]) % inp @@ -1191,6 +1192,7 @@ def test_templating_constants(self): self.assertEqual(ec['modloadmsg'], expected) self.assertEqual(ec['modunloadmsg'], expected) self.assertEqual(ec['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) + self.assertEqual(ec['modextrapaths_append'], {'PATH_APPEND': 'appended_path'}) self.assertEqual(ec['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) # test the escaping insanity here (ie all the crap we allow in easyconfigs) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index cc84ce86d1..e5469dd488 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -285,6 +285,7 @@ def test_toy_tweaked(self): ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}", + "modextrapaths_append = {'SOMEPATH_APPEND': ['qux/fred', 'thud', '']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = '%s'" % modloadmsg, "modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua @@ -319,6 +320,9 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/qux/fred$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/thud$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root$', toy_module_txt, re.M)) mod_load_msg = r'module-info mode load.*\n\s*puts stderr\s*.*%s$' % modloadmsg_regex_tcl self.assertTrue(re.search(mod_load_msg, toy_module_txt, re.M)) self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) @@ -326,6 +330,11 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) pattern = r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$' self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append_path\("SOMEPATH_APPEND", root\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M)) mod_load_msg = r'^if mode\(\) == "load" then\n\s*io.stderr:write\(%s\)$' % modloadmsg_regex_lua From 4e0a068e79d223390cd5fec443cc9cc7d0b5c91b Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 10:20:57 +0100 Subject: [PATCH 030/171] fix tests --- test/framework/toy_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index e5469dd488..0349652b47 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1523,6 +1523,9 @@ def test_toy_module_fulltxt(self): r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)', r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)', r'prepend_path\("SOMEPATH", root\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)', + r'append_path\("SOMEPATH_APPEND", root\)', r'', r'if mode\(\) == "load" then', ] + modloadmsg_lua + [ @@ -1561,6 +1564,9 @@ def test_toy_module_fulltxt(self): r'prepend-path SOMEPATH \$root/foo/bar', r'prepend-path SOMEPATH \$root/baz', r'prepend-path SOMEPATH \$root', + r'append-path SOMEPATH_APPEND \$root/qux/fred', + r'append-path SOMEPATH_APPEND \$root/thud', + r'append-path SOMEPATH_APPEND \$root', r'', r'if { \[ module-info mode load \] } {', ] + modloadmsg_tcl + [ From 2eef785818183d8fd3d53423d0e723729c810460 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 13:13:41 +0100 Subject: [PATCH 031/171] try to fix tests --- test/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index df40f42d98..27ece36ca5 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -644,8 +644,8 @@ def test_make_module_extra(self): installver = '3.14-gompi-2018a' # also check how absolute paths specified in modexself.contents = '\n'.join([ - self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" - self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -677,9 +677,9 @@ def test_make_module_extra(self): r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path", r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar", - r"^append[-_]path.*TEST_PATH_VAR.*root.*foo", - r"^append[-_]path.*TEST_PATH_VAR.*/test/absolute/path", - r"^append[-_]path.*TEST_PATH_VAR.*root.*bar", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*foo", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*/test/absolute/path", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*bar", ] for pattern in patterns: self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt)) @@ -1178,7 +1178,7 @@ def test_make_module_step(self): 'PATH': ('xbin', 'pibin'), 'CPATH': 'pi/include', } - modextrapaths_append = modextrapaths.copy() + modextrapaths_append = {'APPEND_PATH': 'append_path'} self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, From 58d67dbed338aa42ca5fa2076cfe5f0a96eccb21 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 12 Jan 2024 15:56:09 +0100 Subject: [PATCH 032/171] fix --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 27ece36ca5..6c08f947d9 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -644,7 +644,7 @@ def test_make_module_extra(self): installver = '3.14-gompi-2018a' # also check how absolute paths specified in modexself.contents = '\n'.join([ - self.contents += "\nmodextrapaths = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) From 571b94dec403a7afd8e4463fe6b1cd8cc59de373 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 15 Jan 2024 15:36:32 +0100 Subject: [PATCH 033/171] Move post install commands after lib64 symlinking --- easybuild/framework/easyblock.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0f1c6d6f3f..c8967e812c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3019,12 +3019,6 @@ def post_install_step(self): - run post install commands if any were specified """ - self.run_post_install_commands() - self.apply_post_install_patches() - self.print_post_install_messages() - - self.fix_shebang() - lib_dir = os.path.join(self.installdir, 'lib') lib64_dir = os.path.join(self.installdir, 'lib64') @@ -3045,6 +3039,12 @@ def post_install_step(self): # create *relative* 'lib' symlink to 'lib64'; symlink('lib64', lib_dir, use_abspath_source=False) + self.run_post_install_commands() + self.apply_post_install_patches() + self.print_post_install_messages() + + self.fix_shebang() + def sanity_check_step(self, *args, **kwargs): """ Do a sanity check on the installation From b03f26e9ebaa9de6407b1e40459c1d586c3c97d1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 15 Jan 2024 16:44:10 +0100 Subject: [PATCH 034/171] Add script for updating local git repos with develop branch Usefull when using `--git-working-dirs-path` and/or with checkouts created by `install-EasyBuild-develop.sh`. --- .../scripts/install-EasyBuild-develop.sh | 9 +- easybuild/scripts/update-EasyBuild-develop.sh | 91 +++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) create mode 100755 easybuild/scripts/update-EasyBuild-develop.sh diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh index 4181d8c42a..f88ae280a0 100755 --- a/easybuild/scripts/install-EasyBuild-develop.sh +++ b/easybuild/scripts/install-EasyBuild-develop.sh @@ -14,7 +14,7 @@ print_usage() echo echo " github_username: username on GitHub for which the EasyBuild repositories should be cloned" echo - echo " install_dir: directory were all the EasyBuild files will be installed" + echo " install_dir: directory where all the EasyBuild files will be installed" echo } @@ -79,7 +79,7 @@ EOF # Check for 'help' argument -if [ "$1" = "-h" -o "$1" = "--help" ] ; then +if [ "$1" = "-h" ] || [ "$1" = "--help" ] ; then print_usage exit 0 fi @@ -116,13 +116,14 @@ github_clone_branch "easybuild" "develop" EB_DEVEL_MODULE_NAME="EasyBuild-develop" MODULES_INSTALL_DIR=${INSTALL_DIR}/modules EB_DEVEL_MODULE="${MODULES_INSTALL_DIR}/${EB_DEVEL_MODULE_NAME}" -mkdir -p ${MODULES_INSTALL_DIR} +mkdir -p "${MODULES_INSTALL_DIR}" print_devel_module > "${EB_DEVEL_MODULE}" -echo +echo echo "=== Run 'module use ${MODULES_INSTALL_DIR}' and 'module load ${EB_DEVEL_MODULE_NAME}' to use your development version of EasyBuild." echo "=== (you can append ${MODULES_INSTALL_DIR} to your MODULEPATH to make this module always available for loading)" echo echo "=== To update each repository, run 'git pull origin' in each subdirectory of ${INSTALL_DIR}" +echo "=== Or run $(dirname "$0")/update-EasyBuild-develop.sh '${INSTALL_DIR}'" echo exit 0 diff --git a/easybuild/scripts/update-EasyBuild-develop.sh b/easybuild/scripts/update-EasyBuild-develop.sh new file mode 100755 index 0000000000..c80289ad96 --- /dev/null +++ b/easybuild/scripts/update-EasyBuild-develop.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Stop in case of error +set -e + +# Print script help +print_usage() +{ + echo "Checkout develop branch of all EasyBuild repositories" + echo "and pull changes from the remote repository." + echo "To be used with the EasyBuild-develop module or a set git-working-dirs-path" + echo "Usage: $0 []" + echo + echo " git_dir: directory where all the EasyBuild repositories are installed." + echo " Automatically detected if not specified." + echo +} + +if [[ "$1" = "-h" ]] || [[ "$1" = "--help" ]]; then + print_usage + exit 0 +fi + +if [[ $# -gt 1 ]] ; then + echo "Error: invalid arguments" + echo + print_usage + exit 1 +fi + +if [[ $# -eq 1 ]]; then + git_dir=$1 +else + # Auto detect git_dir + git_dir="" + if ! which eb &> /dev/null; then + module load EasyBuild-develop || module load EasyBuild || true + if ! which eb &> /dev/null; then + echo 'Found neither the `eb` command nor a working module.' + echo 'Please specify the git_dir!' + exit 1 + fi + fi + if out=$(eb --show-config | grep -F 'git-working-dirs-path'); then + path=$(echo "$out" | awk '{print $NF}') + if [[ -n "$path" ]] && [[ -d "$path" ]]; then + git_dir=$path + echo "Using git_dir from git-working-dirs-path: $git_dir" + fi + fi + if [[ -z "$git_dir" ]]; then + eb_dir=$(dirname "$(which eb)") + if [[ "$(basename "$eb_dir")" == "easybuild-framework" ]] && [[ -d "$eb_dir/.git" ]]; then + git_dir=$(dirname "$eb_dir") + echo "Using git_dir from eb command: $git_dir" + else + echo 'Please specify the git_dir as auto-detection failed!' + exit 1 + fi + fi +fi + +cd "$git_dir" + +for folder in easybuild easybuild-framework easybuild-easyblocks easybuild-easyconfigs; do + echo # A newline + if [[ -d "$folder" ]]; then + echo "========= Checking ${folder} =========" + else + echo "========= Skipping non-existent ${folder} =========" + fi + cd "$folder" + git checkout "develop" + if git remote | grep -qF github_easybuilders; then + git pull "github_easybuilders" + else + git pull + fi + cd .. +done + +index_file="$git_dir/easybuild-easyconfigs/easybuild/easyconfigs/.eb-path-index" +if [[ -f "$index_file" ]]; then + echo -n "Trying to remove index from ${index_file}..." + if rm "$index_file"; then + echo "Done!" + echo "Recreate with 'eb --create-index \"$(dirname "$index_file")\"'" + else + echo "Failed!" + fi +fi From edeb30a194b09131a0d8785dff77a4547ff601ee Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:05:36 +0000 Subject: [PATCH 035/171] Deprecate EnvironmentModulesC and EnvironmentModulesTcl --- easybuild/tools/modules.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index c0d8c25c59..bdf764e9cc 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1185,6 +1185,7 @@ class EnvironmentModulesC(ModulesTool): COMMAND = "modulecmd" REQ_VERSION = '3.2.10' MAX_VERSION = '3.99' + DEPR_VERSION = '3.999' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' def run_module(self, *args, **kwargs): @@ -1246,6 +1247,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None + DEPR_VERSION = '9999' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From 289ff54b64b90afce798a9d8bdf34b75251f1009 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:12:18 +0000 Subject: [PATCH 036/171] update deprecation message --- easybuild/tools/modules.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index bdf764e9cc..2df2e165e9 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -272,11 +272,7 @@ def set_and_check_version(self): if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version - - if self.version.startswith('6') and 'Lmod6' in build_option('silence_deprecation_warnings'): - self.log.warning(depr_msg) - else: - self.log.deprecated(depr_msg, '5.0') + self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) From bd99f6578967e28a8a1cbc99d1f9f5029cd054cd Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 16 Jan 2024 10:14:03 +0100 Subject: [PATCH 037/171] Error when multiple PR options are passed Passing `--preview-pr` in addition to `--new-pr` will not show a preview but actually create a new PR. In general it is always wrong to pass multiple PR-options to EB. So check for that and error out with an appropriate error message. --- easybuild/main.py | 33 +++++++++++++++++++++++---------- test/framework/options.py | 10 ++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 73550e3998..802009166e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -333,11 +333,23 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session categorized_paths = categorize_files_by_type(eb_args) + pr_options = [ + 'new_branch_github', + 'new_pr', + 'new_pr_from_branch', + 'preview_pr', + 'sync_branch_with_develop', + 'sync_pr_with_develop', + 'update_branch_github', + 'update_pr', + ] + set_pr_options = [opt for opt in pr_options if getattr(options, opt)] + any_pr_option_set = len(set_pr_options) > 0 + if len(set_pr_options) > 1: + raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", + ', '.join(['--' + opt.replace('_', '-') for opt in set_pr_options])) # command line options that do not require any easyconfigs to be specified - pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr - pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop - pr_options = pr_options or options.update_branch_github or options.update_pr - no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query] + no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) @@ -427,9 +439,10 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules - keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec - keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only - keep_available_modules = keep_available_modules or options.inject_checksums_to_json + keep_available_modules = any(( + forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums, + options.sanity_check_only, options.inject_checksums_to_json) + ) # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: @@ -448,12 +461,12 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled - if options.robot and (not dry_run_mode or pr_options): + if options.robot and (not dry_run_mode or any_pr_option_set): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs - elif pr_options: + elif any_pr_option_set: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) @@ -472,7 +485,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return True # creating/updating PRs - if pr_options: + if any_pr_option_set: if options.new_pr: new_pr(categorized_paths, ordered_ecs) elif options.new_branch_github: diff --git a/test/framework/options.py b/test/framework/options.py index 4b5e1afce8..0067525bde 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3915,6 +3915,16 @@ def test_github_review_pr(self): self.mock_stderr(False) self.assertNotIn("2016.04", txt) + def test_set_multiple_pr_opts(self): + """Test that passing multiple PR options results in an error""" + test_cases = [ + ['--new-pr', 'dummy.eb', '--preview-pr'], + ['--new-pr', 'dummy.eb', '--update-pr', '42'], + ] + for args in test_cases: + error_pattern = "The following options are set but incompatible.* " + args[0] + self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) + def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() From 1da0e55c256d63fb75fce06772b39cff270e54cf Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 16 Jan 2024 10:25:08 +0100 Subject: [PATCH 038/171] Remove pr_options variable Use a tuple in the list generator to avoid accidentally using that variable which now contained ALL options not if any one was set (as before). --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 802009166e..90ee5882fc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -333,7 +333,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session categorized_paths = categorize_files_by_type(eb_args) - pr_options = [ + set_pr_options = [opt for opt in ( 'new_branch_github', 'new_pr', 'new_pr_from_branch', @@ -342,8 +342,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session 'sync_pr_with_develop', 'update_branch_github', 'update_pr', + ) if getattr(options, opt) ] - set_pr_options = [opt for opt in pr_options if getattr(options, opt)] any_pr_option_set = len(set_pr_options) > 0 if len(set_pr_options) > 1: raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", From a7961568c82c945f4546aa07d88e9841378fb819 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Jan 2024 12:56:33 +0100 Subject: [PATCH 039/171] don't hardcode /bin/bash in eb script, RPATH wrapper script, and run_shell_cmd --- easybuild/scripts/rpath_wrapper_template.sh.in | 2 +- easybuild/tools/run.py | 5 ++++- eb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 55c3388c5b..73eb21f0e0 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2016-2023 Ghent University # diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0bb0d3c9f4..92c92a1b41 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -41,6 +41,7 @@ import os import re import signal +import shutil import subprocess import sys import tempfile @@ -276,7 +277,9 @@ def to_cmd_str(cmd): # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh) # stick to None (default value) when not running command via a shell if use_bash: - executable, shell = '/bin/bash', True + bash = shutil.which('bash') + _log.info(f"Path to bash that will be used to run shell commands: {bash}") + executable, shell = bash, True else: executable, shell = None, False diff --git a/eb b/eb index 402bb87d0a..f427c5e7e7 100755 --- a/eb +++ b/eb @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2009-2023 Ghent University # From 0b68c30ab63f3ce8ed21adb3eb33105c60b2adf2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 17 Jan 2024 14:06:53 +0100 Subject: [PATCH 040/171] fix broken test_extensions_sanity_check which was hardcoding /bin/bash in expected error message --- test/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 9f656573d1..40ed5f692a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -2082,7 +2082,7 @@ def test_extensions_sanity_check(self): eb.silent = True error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n" error_pattern += r"failing sanity check for 'toy' extension: " - error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash:.* thisshouldfail: command not found' + error_pattern += r'command "thisshouldfail" failed; output:\n.* thisshouldfail: command not found' with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True) From a468ef9f3eb083da8e97285f720bc5e58f28f061 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jan 2024 17:32:02 +0100 Subject: [PATCH 041/171] Enhance error message --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 90ee5882fc..85501f57f0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -346,7 +346,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session ] any_pr_option_set = len(set_pr_options) > 0 if len(set_pr_options) > 1: - raise EasyBuildError("The following options are set but incompatible: %s.\nRemove at least one!", + raise EasyBuildError("The following options are set but incompatible: %s.\nYou can only use one at a time!", ', '.join(['--' + opt.replace('_', '-') for opt in set_pr_options])) # command line options that do not require any easyconfigs to be specified no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] From 17bac3147b220b586b2dadd0b0cb4817f2e01c8e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 17 Jan 2024 17:34:11 +0100 Subject: [PATCH 042/171] Add more test cases --- test/framework/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 0067525bde..f81d34fff8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3920,6 +3920,8 @@ def test_set_multiple_pr_opts(self): test_cases = [ ['--new-pr', 'dummy.eb', '--preview-pr'], ['--new-pr', 'dummy.eb', '--update-pr', '42'], + ['--new-pr', 'dummy.eb', '--sync-pr-with-develop', '42'], + ['--new-pr', 'dummy.eb', '--new-pr-from-branch', 'mybranch'], ] for args in test_cases: error_pattern = "The following options are set but incompatible.* " + args[0] From 8aaaec2904ca2a260e0d4501f6d8da5b553f4af9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 19:13:25 +0100 Subject: [PATCH 043/171] replace string_type with str in easyblock.py --- easybuild/framework/easyblock.py | 2 +- test/framework/easyblock.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index edf4b01f8a..f6077dd5ce 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1413,7 +1413,7 @@ def make_module_extra(self, altroot=None, altversion=None): lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path'])) for (key, value) in self.cfg['modextrapaths_append'].items(): - if isinstance(value, string_type): + if isinstance(value, str): value = [value] elif not isinstance(value, (tuple, list)): raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple", diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 3eed2986b2..64c554ca89 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1273,7 +1273,7 @@ def test_make_module_step(self): self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) for (key, vals) in modextrapaths_append.items(): - if isinstance(vals, string_type): + if isinstance(vals, str): vals = [vals] for val in vals: if get_module_syntax() == 'Tcl': From 7cba1dcf64fdb750a95bc0371a72300826014c1c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 21:40:28 +0100 Subject: [PATCH 044/171] implement initial support for running shell commands asynchronously using run_shell_cmd --- easybuild/tools/run.py | 40 +++++++++------ test/framework/run.py | 95 +++++++++++++++++++++++++++++------ test/framework/systemtools.py | 4 +- 3 files changed, 107 insertions(+), 32 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 92c92a1b41..c8d0cabf71 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -45,6 +45,7 @@ import subprocess import sys import tempfile +import threading import time from collections import namedtuple from datetime import datetime @@ -79,7 +80,7 @@ RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', - 'out_file', 'err_file')) + 'out_file', 'err_file', 'thread_id')) class RunShellCmdError(BaseException): @@ -199,7 +200,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param use_bash: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) - :param asynchronous: run command asynchronously + :param asynchronous: indicate that command is being run asynchronously :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions @@ -223,9 +224,6 @@ def to_cmd_str(cmd): return cmd_str # temporarily raise a NotImplementedError until all options are implemented - if asynchronous: - raise NotImplementedError - if qa_patterns or qa_wait_patterns: raise NotImplementedError @@ -235,6 +233,11 @@ def to_cmd_str(cmd): cmd_str = to_cmd_str(cmd) cmd_name = os.path.basename(cmd_str.split(' ')[0]) + thread_id = None + if asynchronous: + thread_id = threading.get_native_id() + _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}") + # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled") @@ -259,16 +262,16 @@ def to_cmd_str(cmd): if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: silent = build_option('silent') - msg = f" running command \"{cmd_str}\"\n" + msg = f" running shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) start_time = datetime.now() if not hidden: - cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -293,7 +296,11 @@ def to_cmd_str(cmd): stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - _log.info(f"Running command '{cmd_str}' in {work_dir}") + log_msg = f"Running shell command '{cmd_str}' in {work_dir}" + if thread_id: + log_msg += f" (via thread with ID {thread_id})" + _log.info(log_msg) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, cwd=work_dir, env=env, shell=shell, executable=executable) @@ -337,7 +344,7 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) # always log command output cmd_name = cmd_str.split(' ')[0] @@ -370,7 +377,7 @@ def to_cmd_str(cmd): return res -def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): +def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id): """ Helper function to construct and print trace message for command being run @@ -380,11 +387,18 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): :param stdin: stdin input value for command :param cmd_out_fp: path to output file for command :param cmd_err_fp: path to errors/warnings output file for command + :param thread_id: thread ID (None when not running shell command asynchronously) """ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + if thread_id: + run_cmd_msg = f"running shell command (asynchronously, thread ID: {thread_id}):" + else: + run_cmd_msg = "running shell command:" + lines = [ - "running command:", + run_cmd_msg, + f"\t{cmd}", f"\t[started at: {start_time}]", f"\t[working dir: {work_dir}]", ] @@ -395,8 +409,6 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp): if cmd_err_fp: lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]") - lines.append('\t' + cmd) - trace_msg('\n'.join(lines)) diff --git a/test/framework/run.py b/test/framework/run.py index db74940aec..9a9b02d767 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -41,6 +41,7 @@ import tempfile import textwrap import time +from concurrent.futures import ThreadPoolExecutor from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.base.fancylogger import setLogLevelDebug @@ -248,7 +249,7 @@ def test_run_shell_cmd_log(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) - regex_start_cmd = re.compile("Running command 'echo hello' in /") + regex_start_cmd = re.compile("Running shell command 'echo hello' in /") regex_cmd_exit = re.compile(r"Shell command completed successfully \(see output above\): echo hello") # command output is always logged @@ -448,7 +449,7 @@ def test_run_cmd_work_dir(self): def test_run_shell_cmd_work_dir(self): """ - Test running command in specific directory with run_shell_cmd function. + Test running shell command in specific directory with run_shell_cmd function. """ orig_wd = os.getcwd() self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) @@ -615,11 +616,11 @@ def test_run_shell_cmd_trace(self): """Test run_shell_cmd function in trace mode, and with tracing disabled.""" pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -675,11 +676,11 @@ def test_run_shell_cmd_trace_stdin(self): init_config(build_options={'trace': True}) pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\techo hello", r" >> command completed: exit 0, ran in .*", ] @@ -707,8 +708,8 @@ def test_run_shell_cmd_trace_stdin(self): self.assertEqual(res.output, 'hello') self.assertEqual(res.exit_code, 0) self.assertEqual(stderr, '') - pattern.insert(3, r"\t\[input: hello\]") - pattern[-2] = "\tcat" + pattern.insert(4, r"\t\[input: hello\]") + pattern[1] = "\tcat" regex = re.compile('\n'.join(pattern)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) @@ -909,7 +910,8 @@ def test_run_shell_cmd_cache(self): # inject value into cache to check whether executing command again really returns cached value with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, - work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None) + work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, + thread_id=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -928,7 +930,8 @@ def test_run_shell_cmd_cache(self): # inject different output for cat with 'foo' as stdin to check whether cached value is used with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, - work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None) + work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, + thread_id=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) @@ -1006,7 +1009,7 @@ def test_run_shell_cmd_dry_run(self): self.assertEqual(res.output, '') self.assertEqual(res.stderr, None) # check dry run output - expected = """ running command "somecommand foo 123 bar"\n""" + expected = """ running shell command "somecommand foo 123 bar"\n""" self.assertIn(expected, stdout) # check enabling 'hidden' @@ -1029,7 +1032,7 @@ def test_run_shell_cmd_dry_run(self): fail_on_error=False, in_dry_run=True) stdout = self.get_stdout() self.mock_stdout(False) - self.assertNotIn('running command "', stdout) + self.assertNotIn('running shell command "', stdout) self.assertNotEqual(res.exit_code, 0) self.assertEqual(res.output, 'done\n') self.assertEqual(res.stderr, None) @@ -1207,7 +1210,7 @@ def test_run_cmd_async(self): "for i in $(seq 1 50)", "do sleep 0.1", "for j in $(seq 1000)", - "do echo foo", + "do echo foo${i}${j}", "done", "done", "echo done", @@ -1257,8 +1260,68 @@ def test_run_cmd_async(self): res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res['done'], True) self.assertEqual(res['exit_code'], 0) - self.assertTrue(res['output'].startswith('start\n')) - self.assertTrue(res['output'].endswith('\ndone\n')) + self.assertEqual(len(res['output']), 435661) + self.assertTrue(res['output'].startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res['output']) + self.assertTrue(res['output'].endswith('\nfoo501000\ndone\n')) + + def test_run_shell_cmd_async(self): + """Test asynchronously running of a shell command via run_shell_cmd """ + + thread_pool = ThreadPoolExecutor() + + os.environ['TEST'] = 'test123' + env = os.environ.copy() + + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) + + # change value of $TEST to check that command is completed with correct environment + os.environ['TEST'] = 'some_other_value' + + # initial poll should result in None, since it takes a while for the command to complete + self.assertEqual(task.done(), False) + + # wait until command is done + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'sleeping...\ntest123\n') + + # check asynchronous running of failing command + error_test_cmd = "echo 'FAIL!' >&2; exit 123" + task = thread_pool.submit(run_shell_cmd, error_test_cmd, hidden=True, fail_on_error=False, asynchronous=True) + time.sleep(1) + res = task.result() + self.assertEqual(res.exit_code, 123) + self.assertEqual(res.output, "FAIL!\n") + self.assertTrue(res.thread_id) + + # also test with a command that produces a lot of output, + # since that tends to lock up things unless we frequently grab some output... + verbose_test_cmd = ';'.join([ + "echo start", + "for i in $(seq 1 50)", + "do sleep 0.1", + "for j in $(seq 1000)", + "do echo foo${i}${j}", + "done", + "done", + "echo done", + ]) + task = thread_pool.submit(run_shell_cmd, verbose_test_cmd, hidden=True, asynchronous=True) + + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(len(res.output), 435661) + self.assertTrue(res.output.startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res.output) + self.assertTrue(res.output.endswith('\nfoo501000\ndone\n')) def test_check_log_for_errors(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') @@ -1373,7 +1436,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): def test_run_shell_cmd_with_hooks(self): """ - Test running command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. + Test running shell command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. """ cwd = os.getcwd() diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 5a8d1033a5..5f4b62f09c 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): } if cmd in known_cmds: return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), - out_file=None, err_file=None) + out_file=None, err_file=None, thread_id=None) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,7 @@ def test_gcc_version_darwin(self): out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, - out_file=None, err_file=None) + out_file=None, err_file=None, thread_id=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From 1c7c8b4f668dd5d9908bfb5cf92a8d88faef20f6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:07:41 +0100 Subject: [PATCH 045/171] use ThreadPoolExecutor to asynchronously run shell commands in EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 30 ++++++++++++++++++------------ test/framework/toy_build.py | 6 +++--- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f6077dd5ce..9744397b6b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,6 +52,7 @@ import tempfile import time import traceback +from concurrent.futures import ThreadPoolExecutor from datetime import datetime import easybuild.tools.environment as env @@ -87,7 +88,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, check_async_cmd, run_cmd, run_shell_cmd +from easybuild.tools.run import RunShellCmdError, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1818,7 +1819,9 @@ def skip_extensions_parallel(self, exts_filter): self.log.experimental("Skipping installed extensions in parallel") print_msg("skipping installed extensions (in parallel)", log=self.log) - async_cmd_info_cache = {} + thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) + + async_shell_cmd_tasks = {} running_checks_ids = [] installed_exts_ids = [] exts_queue = list(enumerate(self.ext_instances[:])) @@ -1831,14 +1834,14 @@ def skip_extensions_parallel(self, exts_filter): # first handle completed checks for idx in running_checks_ids[:]: ext_name = self.ext_instances[idx].name - # don't read any output, just check whether command completed - async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False) - if async_cmd_info['done']: - out, ec = async_cmd_info['output'], async_cmd_info['exit_code'] - self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out) + # check whether command completed + task = async_shell_cmd_tasks[idx] + if task.done(): + res = task.result() + self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") running_checks_ids.remove(idx) - if ec == 0: - print_msg("skipping extension %s" % ext_name, log=self.log) + if res.exit_code == 0: + print_msg(f"skipping extension {ext_name}", log=self.log) installed_exts_ids.append(idx) checked_exts_cnt += 1 @@ -1847,11 +1850,12 @@ def skip_extensions_parallel(self, exts_filter): self.update_exts_progress_bar(exts_pbar_label) # start additional checks asynchronously - while exts_queue and len(running_checks_ids) < self.cfg['parallel']: + while exts_queue: idx, ext = exts_queue.pop(0) cmd, stdin = resolve_exts_filter_template(exts_filter, ext) - async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, - regexp=False, trace=False, asynchronous=True) + task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, + fail_on_error=False, asynchronous=True) + async_shell_cmd_tasks[idx] = task running_checks_ids.append(idx) # compose new list of extensions, skip over the ones that are already installed; @@ -1864,6 +1868,8 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances + thread_pool.shutdown() + def install_extensions(self, install=True): """ Install extensions. diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6ba47d3587..936fd09c4c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1917,7 +1917,7 @@ def test_toy_exts_parallel(self): write_file(test_ec, test_ec_txt) args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, @@ -1936,7 +1936,7 @@ def test_toy_exts_parallel(self): # also test skipping of extensions in parallel args.append('--skip') - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # order in which these patterns occur is not fixed, so check them one by one @@ -1962,7 +1962,7 @@ def test_toy_exts_parallel(self): write_file(toy_ext_eb, toy_ext_eb_txt) args[-1] = '--include-easyblocks=%s' % toy_ext_eb - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, # in case no progress was made between checks From e91dbfabc4a4eae9500ef4981f9671b53ab0831f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:34:38 +0100 Subject: [PATCH 046/171] use concurrent.futures.wait in EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 44 +++++++++++++++----------------- easybuild/tools/run.py | 8 +++--- test/framework/run.py | 4 +-- test/framework/systemtools.py | 4 +-- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9744397b6b..4f45dbbaab 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -41,7 +41,6 @@ * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) """ - import copy import glob import inspect @@ -52,7 +51,7 @@ import tempfile import time import traceback -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from datetime import datetime import easybuild.tools.environment as env @@ -1821,42 +1820,41 @@ def skip_extensions_parallel(self, exts_filter): thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) - async_shell_cmd_tasks = {} - running_checks_ids = [] + async_shell_cmd_tasks = [] installed_exts_ids = [] exts_queue = list(enumerate(self.ext_instances[:])) checked_exts_cnt = 0 exts_cnt = len(self.ext_instances) + done_tasks = [] # asynchronously run checks to see whether extensions are already installed - while exts_queue or running_checks_ids: + while exts_queue or async_shell_cmd_tasks: # first handle completed checks - for idx in running_checks_ids[:]: + for task in done_tasks: + async_shell_cmd_tasks.remove(task) + res = task.result() + idx = res.task_id ext_name = self.ext_instances[idx].name - # check whether command completed - task = async_shell_cmd_tasks[idx] - if task.done(): - res = task.result() - self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") - running_checks_ids.remove(idx) - if res.exit_code == 0: - print_msg(f"skipping extension {ext_name}", log=self.log) - installed_exts_ids.append(idx) - - checked_exts_cnt += 1 - exts_pbar_label = "skipping installed extensions " - exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) - self.update_exts_progress_bar(exts_pbar_label) + self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") + if res.exit_code == 0: + print_msg(f"skipping extension {ext_name}", log=self.log) + installed_exts_ids.append(idx) + + checked_exts_cnt += 1 + exts_pbar_label = "skipping installed extensions " + exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) + self.update_exts_progress_bar(exts_pbar_label) # start additional checks asynchronously while exts_queue: idx, ext = exts_queue.pop(0) cmd, stdin = resolve_exts_filter_template(exts_filter, ext) task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, - fail_on_error=False, asynchronous=True) - async_shell_cmd_tasks[idx] = task - running_checks_ids.append(idx) + fail_on_error=False, asynchronous=True, task_id=idx) + async_shell_cmd_tasks.append(task) + + (done_tasks, _) = wait(async_shell_cmd_tasks, timeout=1, return_when=FIRST_COMPLETED) # compose new list of extensions, skip over the ones that are already installed; # note: original order in extensions list should be preserved! diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c8d0cabf71..852d13a966 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -80,7 +80,7 @@ RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', - 'out_file', 'err_file', 'thread_id')) + 'out_file', 'err_file', 'thread_id', 'task_id')) class RunShellCmdError(BaseException): @@ -184,7 +184,7 @@ def cache_aware_func(cmd, *args, **kwargs): @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, - output_file=True, stream_output=None, asynchronous=False, with_hooks=True, + output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -267,7 +267,7 @@ def to_cmd_str(cmd): dry_run_msg(msg, silent=silent) return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) start_time = datetime.now() if not hidden: @@ -344,7 +344,7 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, - out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id) + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) # always log command output cmd_name = cmd_str.split(' ')[0] diff --git a/test/framework/run.py b/test/framework/run.py index 9a9b02d767..c86d8635ed 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -911,7 +911,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, - thread_id=None) + thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, None): cached_res}) res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) @@ -931,7 +931,7 @@ def test_run_shell_cmd_cache(self): with self.mocked_stdout_stderr(): cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, - thread_id=None) + thread_id=None, task_id=None) run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 5f4b62f09c..6d8395a5fc 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -341,7 +341,7 @@ def mocked_run_shell_cmd(cmd, **kwargs): } if cmd in known_cmds: return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), - out_file=None, err_file=None, thread_id=None) + out_file=None, err_file=None, thread_id=None, task_id=None) else: return run_shell_cmd(cmd, **kwargs) @@ -774,7 +774,7 @@ def test_gcc_version_darwin(self): out = "Apple LLVM version 7.0.0 (clang-700.1.76)" cwd = os.getcwd() mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, - out_file=None, err_file=None, thread_id=None) + out_file=None, err_file=None, thread_id=None, task_id=None) st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) From 77e077e372c300d698f2f7da7cf005ea7b555977 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 20:00:19 +0100 Subject: [PATCH 047/171] fix error reporting when test step fails --- easybuild/framework/easyblock.py | 5 ++++- test/framework/toy_build.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f6077dd5ce..a684b74347 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2736,7 +2736,10 @@ def _test_step(self): try: self.test_step() except RunShellCmdError as err: - self.report_test_failure(err) + err.print() + ec_path = os.path.basename(self.cfg.path) + error_msg = f"shell command '{err.cmd_name} ...' failed in test step for {ec_path}" + self.report_test_failure(error_msg) def stage_install_step(self): """ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 6ba47d3587..82bc052a93 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -4124,6 +4124,22 @@ def test_toy_build_info_msg(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_toy_failing_test_step(self): + """ + Test behaviour when test step fails, using toy easyconfig. + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nruntest = "false"' + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + error_pattern = r"shell command 'false \.\.\.' failed in test step" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_test_toy_build_with_output, + ec_file=test_ec, raise_error=True) + def test_eb_crash(self): """ Test behaviour when EasyBuild crashes, for example due to a buggy hook From ea562b8ead14500f1112bf4b4b8cbbbf20d3dcd2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 22:45:17 +0100 Subject: [PATCH 048/171] fix broken tests --- test/framework/filetools.py | 46 ++++++++++++++++++------------------- test/framework/run.py | 2 +- test/framework/toy_build.py | 4 ++-- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 800ee9ba0b..8b72aefc4c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2290,7 +2290,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0')) - self.assertTrue(re.search('running command "tar xzf .*/toy-0.0.tar.gz"', txt)) + self.assertTrue(re.search('running shell command "tar xzf .*/toy-0.0.tar.gz"', txt)) with self.mocked_stdout_stderr(): path = ft.extract_file(toy_tarball, self.test_prefix, forced=True, change_into_dir=False) @@ -2314,7 +2314,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertTrue(os.path.samefile(os.getcwd(), self.test_prefix)) self.assertFalse(stderr) - self.assertTrue("running command" in stdout) + self.assertTrue("running shell command" in stdout) # check whether disabling trace output works with self.mocked_stdout_stderr(): @@ -2800,18 +2800,18 @@ def run_check(): } git_repo = {'git_repo': 'git@github.com:easybuilders/testrepository.git'} # Just to make the below shorter expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git test123"', r" \(in /.*\)", ]) % git_repo run_check() @@ -2819,19 +2819,19 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - ' running command "git clone --depth 1 --branch tag_for_tests --recursive' + ' running shell command "git clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2841,11 +2841,11 @@ def run_check(): 'submodule."sha1".active=false', ] expected = '\n'.join([ - ' running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + ' running shell command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + ' clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2854,9 +2854,9 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in /.*\)", - r' running command "tar cfvz .*/target/test.tar.gz testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in /.*\)", ]) % git_repo run_check() @@ -2865,23 +2865,23 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", - r' running command "git checkout 8456f86 && git submodule update --init --recursive"', + r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - ' running command "git checkout 8456f86 && git submodule update --init --recursive' + ' running shell command "git checkout 8456f86 && git submodule update --init --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) % git_repo run_check() @@ -2889,11 +2889,11 @@ def run_check(): del git_config['recursive'] del git_config['recurse_submodules'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in /.*\)", - r' running command "git checkout 8456f86"', + r' running shell command "git checkout 8456f86"', r" \(in /.*/testrepository\)", - r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in /.*\)", ]) % git_repo run_check() diff --git a/test/framework/run.py b/test/framework/run.py index c86d8635ed..1c9b0b2562 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1273,7 +1273,7 @@ def test_run_shell_cmd_async(self): os.environ['TEST'] = 'test123' env = os.environ.copy() - test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + test_cmd = "echo 'sleeping...'; sleep 3; echo $TEST" task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) # change value of $TEST to check that command is completed with correct environment diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 936fd09c4c..6ab83f67fc 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2975,11 +2975,11 @@ def test_toy_build_trace(self): r"^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", r"^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", r'\n'.join([ - r"^ >> running command:", + r"^ >> running shell command:", + r"\tgcc toy.c -o toy\n" r"\t\[started at: .*\]", r"\t\[working dir: .*\]", r"\t\[output saved to .*\]", - r"\tgcc toy.c -o toy\n" r'', ]), r" >> command completed: exit 0, ran in .*", From a19776a5e5896fc3c98f594a6aca410ed941f652 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 23:04:01 +0100 Subject: [PATCH 049/171] use threading.get_ident as fallback for threading.get_native_id for Python < 3.8 --- easybuild/tools/run.py | 10 ++++++++-- test/framework/run.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 852d13a966..986c541538 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -45,11 +45,17 @@ import subprocess import sys import tempfile -import threading import time from collections import namedtuple from datetime import datetime +try: + # get_native_id is only available in Python >= 3.8 + from threading import get_native_id as get_thread_id +except ImportError: + # get_ident is available in Python >= 3.3 + from threading import get_ident as get_thread_id + import easybuild.tools.asyncprocess as asyncprocess from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since @@ -235,7 +241,7 @@ def to_cmd_str(cmd): thread_id = None if asynchronous: - thread_id = threading.get_native_id() + thread_id = get_thread_id() _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}") # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely diff --git a/test/framework/run.py b/test/framework/run.py index 1c9b0b2562..c86d8635ed 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1273,7 +1273,7 @@ def test_run_shell_cmd_async(self): os.environ['TEST'] = 'test123' env = os.environ.copy() - test_cmd = "echo 'sleeping...'; sleep 3; echo $TEST" + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) # change value of $TEST to check that command is completed with correct environment From 75fa3ee8e209beee3e291ad77ddfe0b5101e8475 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:42:28 +0000 Subject: [PATCH 050/171] do not test deprecated module tools --- .github/workflows/unit_tests.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 225b3f85c6..324b64630b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -14,8 +14,6 @@ jobs: runs-on: ubuntu-20.04 outputs: lmod8: Lmod-8.7.6 - modulesTcl: modules-tcl-1.147 - modules3: modules-3.2.10 modules4: modules-4.1.4 steps: - run: "true" @@ -29,8 +27,6 @@ jobs: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - ${{needs.setup.outputs.lmod8}} - - ${{needs.setup.outputs.modulesTcl}} - - ${{needs.setup.outputs.modules3}} - ${{needs.setup.outputs.modules4}} lc_all: [""] include: @@ -156,11 +152,7 @@ jobs: export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH eb --version # tell EasyBuild which modules tool is available - if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl - elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesC - elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then + if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then export EASYBUILD_MODULES_TOOL=EnvironmentModules else export EASYBUILD_MODULES_TOOL=Lmod From fffba9694a1440e560ed36c3201f118405c59a3e Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:05:40 +0000 Subject: [PATCH 051/171] test with non-deprecated module tool --- test/framework/modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index f8226e8d33..b28564885c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1531,10 +1531,10 @@ def test_modulecmd_strip_source(self): '#!/bin/bash', # if last argument (${!#})) is --version, print version 'if [ x"${!#}" == "x--version" ]; then', - ' echo 3.2.10', + ' echo 4.2.10', # otherwise, echo Python commands: set $TEST123 and include a faulty 'source' command 'else', - ' echo "source /opt/cray/pe/modules/3.2.10.6/init/bash"', + ' echo "source /opt/cray/pe/modules/4.2.10.6/init/bash"', " echo \"os.environ['TEST123'] = 'test123'\"", 'fi', ]) @@ -1543,7 +1543,7 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) - modtool = EnvironmentModulesC() + modtool = EnvironmentModules() modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') From 7fffcba6a279a8ea3803f162154810a334236d1f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:08:45 +0000 Subject: [PATCH 052/171] version with a . --- easybuild/tools/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 09524d5ee1..aa8cbf9381 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1243,7 +1243,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None - DEPR_VERSION = '9999' + DEPR_VERSION = '9999.9' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): From fd18449daae861c5c8b522942d2346dd98fb674f Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:00:18 +0000 Subject: [PATCH 053/171] set a DEPR_VERSION for EnvironmentModules --- easybuild/tools/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index aa8cbf9381..75a1022b50 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1318,6 +1318,7 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' + DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' From 8d8029d02fe241e495d2a96311e2c169fb8f0a9d Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:12:57 +0000 Subject: [PATCH 054/171] uses old module tool, so allow deprecated --- test/framework/modules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index b28564885c..f1a242f54a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1531,10 +1531,10 @@ def test_modulecmd_strip_source(self): '#!/bin/bash', # if last argument (${!#})) is --version, print version 'if [ x"${!#}" == "x--version" ]; then', - ' echo 4.2.10', + ' echo 3.2.10', # otherwise, echo Python commands: set $TEST123 and include a faulty 'source' command 'else', - ' echo "source /opt/cray/pe/modules/4.2.10.6/init/bash"', + ' echo "source /opt/cray/pe/modules/3.2.10.6/init/bash"', " echo \"os.environ['TEST123'] = 'test123'\"", 'fi', ]) @@ -1543,7 +1543,8 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) - modtool = EnvironmentModules() + self.allow_deprecated_behaviour() + modtool = EnvironmentModulesC() modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') From cc2a47817e41c6fd62b03ec40976c1a6f6f9897b Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:54:57 +0000 Subject: [PATCH 055/171] and capture the dep. warning --- test/framework/modules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index f1a242f54a..a849148bdf 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1544,8 +1544,9 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) self.allow_deprecated_behaviour() - modtool = EnvironmentModulesC() - modtool.run_module('load', 'test123') + with self.mocked_stdout_stderr(): + modtool = EnvironmentModulesC() + modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') def test_get_setenv_value_from_modulefile(self): From 7315b9f7a6e47515bb255bf2aa0da8cb22595ff2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 24 Jan 2024 14:25:47 +0100 Subject: [PATCH 056/171] improve findPythonDeps to recognize non-canonical package names For e.g. `ruamel.yaml` the canonical name is `ruamel-yaml` but the package name as recorded is still `ruamel.yaml`. So if the search using the canonical name didn't find anything try again with the original name before failing. Tested on `maggma==0.60.2` --- easybuild/scripts/findPythonDeps.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index d6e496a048..c3607e47c7 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -96,10 +96,13 @@ def get_dep_tree(package_spec, verbose): def find_deps(pkgs, dep_tree): """Recursively resolve dependencies of the given package(s) and return them""" res = [] - for pkg in pkgs: - pkg = canonicalize_name(pkg) + for orig_pkg in pkgs: + pkg = canonicalize_name(orig_pkg) matching_entries = [entry for entry in dep_tree if pkg in (entry['package']['package_name'], entry['package']['key'])] + if not matching_entries: + matching_entries = [entry for entry in dep_tree + if orig_pkg in (entry['package']['package_name'], entry['package']['key'])] if not matching_entries: raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree)) if len(matching_entries) > 1: From e91fa2978f485c1a887f674b696f5437beec6124 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 09:52:31 +0100 Subject: [PATCH 057/171] continue to execute deprecated run() methods still found on custom easyblock --- easybuild/framework/easyblock.py | 11 ++++++++++- easybuild/framework/extension.py | 6 +----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3a42b40a0f..9c543ad8f1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1941,7 +1941,16 @@ def install_extensions_sequential(self, install=True): try: ext.pre_install_extension() with self.module_generator.start_module_creation(): - txt = ext.install_extension() + parent_ext_obj = super(ext.__class__, ext) + if ext.run.__hash__() != parent_ext_obj.run.__hash__(): + # DEPRECATED: easyblock has custom run() method + self.log.deprecated( + "Extension.run() is deprecated, use Extension.install_extension() instead.", + '6.0', + ) + txt = ext.run() + else: + txt = ext.install_extension() if txt: self.module_extra_extensions += txt ext.post_install_extension() diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 82f6063a52..171c6265ff 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -189,12 +189,8 @@ def pre_install_extension(self): def run(self, *args, **kwargs): """ - [DEPRECATED] Actual installation of an extension. + [DEPRECATED][6.0] Actual installation of an extension. """ - self.log.deprecated( - "Extension.run() is deprecated, use Extension.install_extension() instead.", - '6.0', - ) self.install_extension(*args, **kwargs) def install_extension(self, *args, **kwargs): From cf5c10b77bf34b5ce791b62f997a70aabba6828d Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 13:00:05 +0100 Subject: [PATCH 058/171] add Extension.install_extension_substep wrapping all substeps to detect and use any custom deprecated methods in the Easyblock --- easybuild/framework/easyblock.py | 21 ++++--------- easybuild/framework/extension.py | 51 +++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 9c543ad8f1..855c386e42 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1939,21 +1939,12 @@ def install_extensions_sequential(self, install=True): # actual installation of the extension if install: try: - ext.pre_install_extension() + ext.install_extension_substep("pre_install_extension") with self.module_generator.start_module_creation(): - parent_ext_obj = super(ext.__class__, ext) - if ext.run.__hash__() != parent_ext_obj.run.__hash__(): - # DEPRECATED: easyblock has custom run() method - self.log.deprecated( - "Extension.run() is deprecated, use Extension.install_extension() instead.", - '6.0', - ) - txt = ext.run() - else: - txt = ext.install_extension() + txt = ext.install_extension_substep("install_extension") if txt: self.module_extra_extensions += txt - ext.post_install_extension() + ext.install_extension_substep("post_install_extension") finally: if not self.dry_run: ext_duration = datetime.now() - start_time @@ -2014,7 +2005,7 @@ def update_exts_progress_bar_helper(running_exts, progress_size): for ext in running_exts[:]: if self.dry_run or ext.async_cmd_check(): self.log.info("Installation of %s completed!", ext.name) - ext.post_install_extension() + ext.install_extension_substep("post_install_extension") running_exts.remove(ext) installed_ext_names.append(ext.name) update_exts_progress_bar_helper(running_exts, 1) @@ -2085,8 +2076,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) if install: - ext.pre_install_extension() - ext.install_extension_async() + ext.install_extension_substep("pre_install_extension") + ext.install_extension_substep("install_extension_async") running_exts.append(ext) self.log.info("Started installation of extension %s in the background...", ext.name) update_exts_progress_bar_helper(running_exts, 0) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 171c6265ff..3073d18165 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -173,12 +173,9 @@ def version(self): def prerun(self): """ - [DEPRECATED] Stuff to do before installing a extension. + [DEPRECATED][6.0] Stuff to do before installing a extension. """ - self.log.deprecated( - "Extension.prerun() is deprecated, use Extension.pre_install_extension() instead.", - '6.0', - ) + # Deprecation warning triggered by Extension.install_extension_substep() self.pre_install_extension() def pre_install_extension(self): @@ -191,6 +188,7 @@ def run(self, *args, **kwargs): """ [DEPRECATED][6.0] Actual installation of an extension. """ + # Deprecation warning triggered by Extension.install_extension_substep() self.install_extension(*args, **kwargs) def install_extension(self, *args, **kwargs): @@ -217,12 +215,9 @@ def install_extension_async(self, *args, **kwargs): def postrun(self): """ - [DEPRECATED] Stuff to do after installing a extension. + [DEPRECATED][6.0] Stuff to do after installing a extension. """ - self.log.deprecated( - "Extension.postrun() is deprecated, use Extension.post_install_extension() instead.", - '6.0', - ) + # Deprecation warning triggered by Extension.install_extension_substep() self.post_install_extension() def post_install_extension(self): @@ -231,6 +226,42 @@ def post_install_extension(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) + def install_extension_substep(self, substep): + """ + Carry out extension installation substep allowing use of deprecated + methods on those extensions using an older EasyBlock + """ + deprecated = { + 'pre_install_extension': 'prerun', + 'install_extension': 'run', + 'install_extension_async': 'run_async', + 'post_install_extension': 'postrun', + } + + if substep not in deprecated: + raise EasyBuildError("Unknown extension installation substep: %s", substep) + + try: + ext_substep = getattr(self, deprecated[substep]) + parent_obj = super(self.__class__, self) + parent_substep = getattr(parent_obj, deprecated[substep]) + except AttributeError: + self.log.debug("Easyblock does not provide deprecated method for installation substep: %s", substep) + ext_substep = getattr(self, substep) + else: + if ext_substep.__hash__() == parent_substep.__hash__(): + # Deprecated method is present in parent, but no custom method in child Easyblock + ext_substep = getattr(self, substep) + else: + # Custom deprecated method used by child Easyblock + self.log.debug("Easyblock provides custom deprecated method for installation substep: %s", substep) + self.log.deprecated( + f"Extension.{deprecated[substep]}() is deprecated, use Extension.{substep}() instead.", + "6.0", + ) + + ext_substep() + def async_cmd_start(self, cmd, inp=None): """ Start installation asynchronously using specified command. From 7cd6732488517af4f9d535a811803dcbeeabccfe Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 18:49:31 +0100 Subject: [PATCH 059/171] Extension.install_extension_substep returns outcome of install extension substep --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 3073d18165..53020d6cf0 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -260,7 +260,7 @@ def install_extension_substep(self, substep): "6.0", ) - ext_substep() + return ext_substep() def async_cmd_start(self, cmd, inp=None): """ From 6074b5511befe2917aa53cece9ef037acfea3d43 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Fri, 26 Jan 2024 18:56:34 +0100 Subject: [PATCH 060/171] remove duplicate deprecation warning on Extension.run_async() --- easybuild/framework/extension.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 53020d6cf0..f807c1f6b1 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -199,12 +199,9 @@ def install_extension(self, *args, **kwargs): def run_async(self, *args, **kwargs): """ - [DEPRECATED] Asynchronous installation of an extension. + [DEPRECATED][6.0] Asynchronous installation of an extension. """ - self.log.deprecated( - "Extension.run_async() is deprecated, use Extension.install_extension_async() instead.", - '6.0', - ) + # Deprecation warning triggered by Extension.install_extension_substep() self.install_extension_async(*args, **kwargs) def install_extension_async(self, *args, **kwargs): From 36febd8739ab37900577db8d0cdd13ea0e1f48fa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 29 Jan 2024 15:18:56 +0100 Subject: [PATCH 061/171] use run_shell_cmd to install extensions in parallel --- easybuild/framework/easyblock.py | 33 ++++++++----- easybuild/framework/extension.py | 48 +------------------ easybuild/tools/run.py | 1 + .../easyblocks/generic/toy_extension.py | 11 +++-- .../sandbox/easybuild/easyblocks/t/toy.py | 6 ++- 5 files changed, 36 insertions(+), 63 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4f45dbbaab..e2d84c162e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -87,7 +87,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import RunShellCmdError, run_shell_cmd +from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -1961,6 +1961,8 @@ def install_extensions_parallel(self, install=True): """ self.log.info("Installing extensions in parallel...") + thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) + running_exts = [] installed_ext_names = [] @@ -1997,16 +1999,23 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # check for extension installations that have completed if running_exts: - self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + self.log.info(f"Checking for completed extension installations ({len(running_exts)} running)...") for ext in running_exts[:]: - if self.dry_run or ext.async_cmd_check(): - self.log.info("Installation of %s completed!", ext.name) - ext.postrun() - running_exts.remove(ext) - installed_ext_names.append(ext.name) - update_exts_progress_bar_helper(running_exts, 1) + if self.dry_run or ext.async_cmd_task.done(): + res = ext.async_cmd_task.result() + if res.exit_code == 0: + self.log.info(f"Installation of extension {ext.name} completed!") + # run post-install method for extension from same working dir as installation of extension + cwd = change_dir(res.work_dir) + ext.postrun() + change_dir(cwd) + running_exts.remove(ext) + installed_ext_names.append(ext.name) + update_exts_progress_bar_helper(running_exts, 1) + else: + raise_run_shell_cmd_error(res) else: - self.log.debug("Installation of %s is still running...", ext.name) + self.log.debug(f"Installation of extension {ext.name} is still running...") # try to start as many extension installations as we can, taking into account number of available cores, # but only consider first 100 extensions still in the queue @@ -2073,9 +2082,9 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.prerun() - ext.run_async() + ext.async_cmd_task = ext.run_async(thread_pool) running_exts.append(ext) - self.log.info("Started installation of extension %s in the background...", ext.name) + self.log.info(f"Started installation of extension {ext.name} in the background...") update_exts_progress_bar_helper(running_exts, 0) # print progress info after every iteration (unless that info is already shown via progress bar) @@ -2088,6 +2097,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + thread_pool.shutdown() + # # MISCELLANEOUS UTILITY FUNCTIONS # diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index d30242495c..9f099eb74c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -42,7 +42,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, run_cmd, run_shell_cmd +from easybuild.tools.run import run_shell_cmd def resolve_exts_filter_template(exts_filter, ext): @@ -150,12 +150,7 @@ def __init__(self, mself, ext, extra_params=None): self.sanity_check_module_loaded = False self.fake_mod_data = None - self.async_cmd_info = None - self.async_cmd_output = None - self.async_cmd_check_cnt = None - # initial read size should be relatively small, - # to avoid hanging for a long time until desired output is available in async_cmd_check - self.async_cmd_read_size = 1024 + self.async_cmd_task = None @property def name(self): @@ -195,44 +190,6 @@ def postrun(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) - def async_cmd_start(self, cmd, inp=None): - """ - Start installation asynchronously using specified command. - """ - self.async_cmd_output = '' - self.async_cmd_check_cnt = 0 - self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) - - def async_cmd_check(self): - """ - Check progress of installation command that was started asynchronously. - - :return: True if command completed, False otherwise - """ - if self.async_cmd_info is None: - raise EasyBuildError("No installation command running asynchronously for %s", self.name) - elif self.async_cmd_info is False: - self.log.info("No asynchronous command was started for extension %s", self.name) - return True - else: - self.log.debug("Checking on installation of extension %s...", self.name) - # use small read size, to avoid waiting for a long time until sufficient output is produced - res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) - self.async_cmd_output += res['output'] - if res['done']: - self.log.info("Installation of extension %s completed!", self.name) - self.async_cmd_info = None - else: - self.async_cmd_check_cnt += 1 - self.log.debug("Installation of extension %s still running (checked %d times)", - self.name, self.async_cmd_check_cnt) - # increase read size after sufficient checks, - # to avoid that installation hangs due to output buffer filling up... - if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): - self.async_cmd_read_size *= 2 - - return res['done'] - @property def required_deps(self): """Return list of required dependencies for this extension.""" @@ -273,7 +230,6 @@ def sanity_check_step(self): self.log.info("modulename set to False for '%s' extension, so skipping sanity check", self.name) elif exts_filter: cmd, stdin = resolve_exts_filter_template(exts_filter, self) - # set log_ok to False so we can catch the error instead of run_cmd cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) if cmd_res.exit_code: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 986c541538..95391f3821 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -207,6 +207,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param output_file: collect command output in temporary output file :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) :param asynchronous: indicate that command is being run asynchronously + :param task_id: task ID for specified shell command (included in return value) :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index de8c4c89c1..15fe5773aa 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -27,6 +27,7 @@ @author: Kenneth Hoste (Ghent University) """ +import os from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock @@ -81,22 +82,24 @@ def prerun(self): super(Toy_Extension, self).run(unpack_src=True) EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) - def run_async(self): + def run_async(self, thread_pool): """ Install toy extension asynchronously. """ + task_id = f'ext_{self.name}_{self.version}' if self.src: cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) else: - self.async_cmd_info = False + cmd = f"echo 'no sources for {self.name}'" + + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ Wrap up installation of toy extension. """ super(Toy_Extension, self).postrun() - EB_toy.install_step(self.master, name=self.name) def sanity_check_step(self, *args, **kwargs): diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index a1454435e3..fb842bfe06 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -163,12 +163,14 @@ def run(self): """ self.build_step() - def run_async(self): + def run_async(self, thread_pool): """ Asynchronous installation of toy as extension. """ cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) + task_id = f'ext_{self.name}_{self.version}' + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ From cedd5ab74a422b2cbaa026d47204a55ee4be540b Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 3 Feb 2024 19:47:46 +0000 Subject: [PATCH 062/171] enable rpath by default --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index be5cf94f75..cb6ec2022f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -489,7 +489,7 @@ def override_options(self): 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), - 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), + 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', True), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", None, 'store', None), From 54b36337a187bf1f6605216d82d2ee455fee5dbc Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:44:39 +0000 Subject: [PATCH 063/171] disable rpath in two tests --- test/framework/options.py | 1 + test/framework/toolchain.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index ca8a582365..5d87695661 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4121,6 +4121,7 @@ def test_extended_dry_run(self): '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', + '--disable-rpath', ] msg_regexs = [ diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 1c8c1771a6..918b344dcf 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2269,6 +2269,7 @@ def test_compiler_cache(self): "--force", "--debug", "--disable-cleanup-tmpdir", + "--disable-rpath", ] ccache = which('ccache') From 0bee2eca2b252d520ab1f24cfe18400718bc9cec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Feb 2024 13:50:38 +0100 Subject: [PATCH 064/171] fix EasyBlock in deprecation log message produced by EasyBlock.install_extensions --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 855c386e42..e6b39c009a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1867,7 +1867,7 @@ def skip_extensions_parallel(self, exts_filter): def install_extensions(self, *args, **kwargs): """[DEPRECATED] Install extensions.""" self.log.deprecated( - "Easyblock.install_extensions() is deprecated, use Easyblock.install_all_extensions() instead.", + "EasyBlock.install_extensions() is deprecated, use EasyBlock.install_all_extensions() instead.", '6.0', ) self.install_all_extensions(*args, **kwargs) @@ -4479,7 +4479,7 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): for easyblock_class in inspect.getmro(type(easyblock_instance)): easyblock_path = inspect.getsourcefile(easyblock_class) # if we reach EasyBlock, Extension or ExtensionEasyBlock class, we are done - # (Extension and ExtensionEasyblock are hardcoded to avoid a cyclical import) + # (Extension and ExtensionEasyBlock are hardcoded to avoid a cyclical import) if easyblock_class.__name__ in [EasyBlock.__name__, 'Extension', 'ExtensionEasyBlock']: break else: From e3845799fef67a0c7215a06905ef5867626dc34c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Feb 2024 13:52:03 +0100 Subject: [PATCH 065/171] enhance install_extension_substep to support passing down (named) arguments --- easybuild/framework/extension.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index f807c1f6b1..a68f22a38d 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -223,27 +223,30 @@ def post_install_extension(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) - def install_extension_substep(self, substep): + def install_extension_substep(self, substep, *args, **kwargs): """ Carry out extension installation substep allowing use of deprecated methods on those extensions using an older EasyBlock """ - deprecated = { + substeps_mapping = { 'pre_install_extension': 'prerun', 'install_extension': 'run', 'install_extension_async': 'run_async', 'post_install_extension': 'postrun', } - if substep not in deprecated: + deprecated_method = substeps_mapping.get(substep) + if deprecated_method is None: raise EasyBuildError("Unknown extension installation substep: %s", substep) try: - ext_substep = getattr(self, deprecated[substep]) + ext_substep = getattr(self, deprecated_method) parent_obj = super(self.__class__, self) - parent_substep = getattr(parent_obj, deprecated[substep]) + parent_substep = getattr(parent_obj, deprecated_method) except AttributeError: - self.log.debug("Easyblock does not provide deprecated method for installation substep: %s", substep) + log_msg = f"EasyBlock does not implement deprecated method '{deprecated_method}' " + log_msg += f"for installation substep {substep}" + self.log.debug(log_msg) ext_substep = getattr(self, substep) else: if ext_substep.__hash__() == parent_substep.__hash__(): @@ -251,13 +254,13 @@ def install_extension_substep(self, substep): ext_substep = getattr(self, substep) else: # Custom deprecated method used by child Easyblock - self.log.debug("Easyblock provides custom deprecated method for installation substep: %s", substep) + self.log.debug(f"EasyBlock provides custom deprecated method for installation substep: {substep}") self.log.deprecated( f"Extension.{deprecated[substep]}() is deprecated, use Extension.{substep}() instead.", "6.0", ) - return ext_substep() + return ext_substep(*args, **kwargs) def async_cmd_start(self, cmd, inp=None): """ From 166227862602b04dfdb75cd64438f30785030056 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 7 Feb 2024 16:56:45 +0100 Subject: [PATCH 066/171] fix variable rename in install_extension_substep error message --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a68f22a38d..b4570f0f75 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -256,7 +256,7 @@ def install_extension_substep(self, substep, *args, **kwargs): # Custom deprecated method used by child Easyblock self.log.debug(f"EasyBlock provides custom deprecated method for installation substep: {substep}") self.log.deprecated( - f"Extension.{deprecated[substep]}() is deprecated, use Extension.{substep}() instead.", + f"Extension.{deprecated_method}() is deprecated, use Extension.{substep}() instead.", "6.0", ) From 92e9fe1ea235dec4b5859c9d6b27354bc763cc9e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Feb 2024 18:11:35 +0100 Subject: [PATCH 067/171] fix broken tests after symlinking lib -> lib64 before postinstallcmds --- test/framework/toy_build.py | 42 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f44182e2b8..f4ee8395c1 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2414,7 +2414,14 @@ def test_sanity_check_paths_lib64(self): # modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt) - postinstallcmd = "mkdir %(installdir)s/lib64 && mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a" + postinstallcmd = ' && '.join([ + # remove lib64 symlink (if it's there) + "rm -f %(installdir)s/lib64", + # create empty lib64 dir + "mkdir %(installdir)s/lib64", + # move libtoy.a + "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a", + ]) ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt) test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') @@ -3829,7 +3836,6 @@ def test_toy_build_lib_lib64_symlink(self): toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') test_ec_txt = read_file(toy_ec) - test_ec_txt += "\npostinstallcmds += ['mv %(installdir)s/lib %(installdir)s/lib64']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -3842,30 +3848,30 @@ def test_toy_build_lib_lib64_symlink(self): lib_path = os.path.join(toy_installdir, 'lib') lib64_path = os.path.join(toy_installdir, 'lib64') - # lib64 subdir exists, is not a symlink - self.assertExists(lib64_path) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) - - # lib subdir is a symlink to lib64 subdir + # lib subdir exists, is not a symlink self.assertExists(lib_path) self.assertTrue(os.path.isdir(lib_path)) - self.assertTrue(os.path.islink(lib_path)) - self.assertTrue(os.path.samefile(lib_path, lib64_path)) + self.assertFalse(os.path.islink(lib_path)) - # lib symlink should point to a relative path - self.assertFalse(os.path.isabs(os.readlink(lib_path))) + # lib64 subdir is a symlink to lib subdir + self.assertExists(lib64_path) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertTrue(os.path.islink(lib64_path)) + self.assertTrue(os.path.samefile(lib64_path, lib_path)) + + # lib64 symlink should point to a relative path + self.assertFalse(os.path.isabs(os.readlink(lib64_path))) # cleanup and try again with --disable-lib-lib64-symlink remove_dir(self.test_installpath) with self.mocked_stdout_stderr(): - self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink']) + self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink']) - self.assertExists(lib64_path) - self.assertNotExists(lib_path) - self.assertNotIn('lib', os.listdir(toy_installdir)) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) + self.assertExists(lib_path) + self.assertNotExists(lib64_path) + self.assertNotIn('lib64', os.listdir(toy_installdir)) + self.assertTrue(os.path.isdir(lib_path)) + self.assertFalse(os.path.islink(lib_path)) def test_toy_build_sanity_check_linked_libs(self): """Test sanity checks for banned/requires libraries.""" From 612b1c0448db922adaed55c1f14ebb45f8bff2a8 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:29:51 +0000 Subject: [PATCH 068/171] rpath off on mac --- easybuild/tools/options.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index cb6ec2022f..197514eb07 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -103,8 +103,8 @@ from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories -from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info +from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family +from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info from easybuild.tools.version import this_is_easybuild @@ -131,6 +131,8 @@ def terminal_supports_colors(stream): DEFAULT_LIST_PR_ORDER = GITHUB_PR_ORDER_CREATED DEFAULT_LIST_PR_DIREC = GITHUB_PR_DIRECTION_DESC +RPATH_DEFAULT = False if get_os_type() == DARWIN else True + _log = fancylogger.getLogger('options', fname=False) @@ -489,7 +491,7 @@ def override_options(self): 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), - 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', True), + 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', RPATH_DEFAULT), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", None, 'store', None), From 44b2b9f6965a9eba1182b57de948847284590b62 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 8 Feb 2024 03:12:14 +0100 Subject: [PATCH 069/171] compare deprecated method qualified name instead of hashes in Extension.install_extension_substep --- easybuild/framework/extension.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index b4570f0f75..a0b9b59ea0 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -235,32 +235,35 @@ def install_extension_substep(self, substep, *args, **kwargs): 'post_install_extension': 'postrun', } - deprecated_method = substeps_mapping.get(substep) - if deprecated_method is None: + deprecated_substep = substeps_mapping.get(substep) + if deprecated_substep is None: raise EasyBuildError("Unknown extension installation substep: %s", substep) try: - ext_substep = getattr(self, deprecated_method) - parent_obj = super(self.__class__, self) - parent_substep = getattr(parent_obj, deprecated_method) + substep_method = getattr(self, deprecated_substep) except AttributeError: - log_msg = f"EasyBlock does not implement deprecated method '{deprecated_method}' " + log_msg = f"EasyBlock does not implement deprecated method '{deprecated_substep}' " log_msg += f"for installation substep {substep}" self.log.debug(log_msg) - ext_substep = getattr(self, substep) + substep_method = getattr(self, substep) else: - if ext_substep.__hash__() == parent_substep.__hash__(): - # Deprecated method is present in parent, but no custom method in child Easyblock - ext_substep = getattr(self, substep) + # Qualified method name contains class defining the method (PEP 3155) + substep_method_name = substep_method.__qualname__ + self.log.debug(f"Found deprecated method in EasyBlock: {substep_method_name}") + + base_method_name = f"Extension.{substep}" + if substep_method_name == base_method_name: + # No custom method in child Easyblock, deprecated method is defined by base Extension class + # Switch to non-deprecated substep method + substep_method = getattr(self, substep) else: # Custom deprecated method used by child Easyblock - self.log.debug(f"EasyBlock provides custom deprecated method for installation substep: {substep}") self.log.deprecated( - f"Extension.{deprecated_method}() is deprecated, use Extension.{substep}() instead.", + f"{substep_method_name}() is deprecated, use {substep}() instead.", "6.0", ) - return ext_substep(*args, **kwargs) + return substep_method(*args, **kwargs) def async_cmd_start(self, cmd, inp=None): """ From 4ded825633071051087c0e5d66b6108b7a219382 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 8 Feb 2024 09:16:13 +0100 Subject: [PATCH 070/171] fix check on deprecated methods on base Extension class --- easybuild/framework/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a0b9b59ea0..19f8a77730 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -251,7 +251,7 @@ def install_extension_substep(self, substep, *args, **kwargs): substep_method_name = substep_method.__qualname__ self.log.debug(f"Found deprecated method in EasyBlock: {substep_method_name}") - base_method_name = f"Extension.{substep}" + base_method_name = f"Extension.{deprecated_substep}" if substep_method_name == base_method_name: # No custom method in child Easyblock, deprecated method is defined by base Extension class # Switch to non-deprecated substep method From 0960bc4f459826e6f378ad963ebe79d1b9474e8e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 8 Feb 2024 13:14:50 +0100 Subject: [PATCH 071/171] clean up log file of EasyBlock instance in check_sha256_checksums --- easybuild/framework/easyconfig/tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index c6649c383b..73393040c6 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -701,7 +701,9 @@ def check_sha256_checksums(ecs, whitelist=None): continue eb_class = get_easyblock_class(ec['easyblock'], name=ec['name']) - checksum_issues.extend(eb_class(ec).check_checksums()) + eb = eb_class(ec) + checksum_issues.extend(eb.check_checksums()) + eb.close_log() return checksum_issues From 22e12b4c068028a34ed2a85424b330bce96a090d Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:43:45 +0000 Subject: [PATCH 072/171] allow only alphanumeric characters in the output filename --- easybuild/tools/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 92c92a1b41..e525f8f95a 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -42,6 +42,7 @@ import re import signal import shutil +import string import subprocess import sys import tempfile @@ -233,7 +234,6 @@ def to_cmd_str(cmd): work_dir = os.getcwd() cmd_str = to_cmd_str(cmd) - cmd_name = os.path.basename(cmd_str.split(' ')[0]) # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely if stream_output is None and build_option('logtostdout'): @@ -244,6 +244,9 @@ def to_cmd_str(cmd): if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) + # restrict the allowed characters in the name of the output_file + allowed_chars = string.ascii_letters + string.digits + cmd_name = ''.join([c for c in os.path.basename(cmd_str.split(' ')[0]) if c in allowed_chars]) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') From faf8f1439b51f388fad600021a316df1acd9ab32 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 15 Feb 2024 11:14:33 +0000 Subject: [PATCH 073/171] functionalise the simplification and add tests --- easybuild/tools/run.py | 17 ++++++++++++++--- test/framework/run.py | 21 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index e525f8f95a..a2854ad8fb 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -181,6 +181,19 @@ def cache_aware_func(cmd, *args, **kwargs): run_shell_cmd_cache = run_cmd_cache +def fileprefix_from_cmd(cmd, allowed_chars=False): + """ + Simplify the cmd to only the allowed_chars we want in a filename + + :param cmd: the cmd (string) + :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits) + """ + if not allowed_chars: + allowed_chars = string.ascii_letters + string.digits + + return ''.join([c for c in cmd if c in allowed_chars]) + + @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, @@ -244,9 +257,7 @@ def to_cmd_str(cmd): if output_file: toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') os.makedirs(toptmpdir, exist_ok=True) - # restrict the allowed characters in the name of the output_file - allowed_chars = string.ascii_letters + string.digits - cmd_name = ''.join([c for c in os.path.basename(cmd_str.split(' ')[0]) if c in allowed_chars]) + cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') cmd_out_fp = os.path.join(tmpdir, 'out.txt') _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') diff --git a/test/framework/run.py b/test/framework/run.py index db74940aec..22fd2a86ba 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -35,6 +35,7 @@ import os import re import signal +import string import stat import subprocess import sys @@ -51,7 +52,7 @@ from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors -from easybuild.tools.run import complete_cmd, get_output_from_process, parse_log_for_error +from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -193,6 +194,24 @@ def test_run_shell_cmd_basic(self): self.assertTrue(isinstance(res.output, str)) self.assertTrue(res.work_dir and isinstance(res.work_dir, str)) + def test_fileprefix_from_cmd(self): + """test simplifications from fileprefix_from_cmd.""" + cmds = { + 'abd123': 'abd123', + 'ab"a': 'aba', + 'a{:$:S@"a': 'aSa', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) + + cmds = { + 'abd123': 'abd', + 'ab"a': 'aba', + '0a{:$:2@"a': 'aa', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd, allowed_chars=string.ascii_letters), expected_simplification) + def test_run_cmd_log(self): """Test logging of executed commands.""" fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') From 8fac144b1ff8960b788b60a883cfce1e85ef3a16 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 16 Feb 2024 10:24:36 +0100 Subject: [PATCH 074/171] fix description of `backup-modules` The documentation is wrong as it doesn't apply to `--module-only` but rather to `--rebuild` or `--force`. Also the auto-enabling is not mentioned. --- easybuild/tools/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 554775be20..0ea04c7672 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -357,7 +357,8 @@ def override_options(self): None, 'store_true', False), 'allow-use-as-root-and-accept-consequences': ("Allow using of EasyBuild as root (NOT RECOMMENDED!)", None, 'store_true', False), - 'backup-modules': ("Back up an existing module file, if any. Only works when using --module-only", + 'backup-modules': ("Back up an existing module file, if any. " + "Auto-enabled when using --module-only or --skip", None, 'store_true', None), # default None to allow auto-enabling if not disabled 'backup-patched-files': ("Create a backup (*.orig) file when applying a patch", None, 'store_true', False), From 6c000d43c26a49f3b714e7faf6c1f0fa1ef253c6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 17 Feb 2024 13:03:53 +0000 Subject: [PATCH 075/171] enhance download instructions --- easybuild/framework/easyblock.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a684b74347..aac45552e7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -53,6 +53,7 @@ import time import traceback from datetime import datetime +from textwrap import indent import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -952,7 +953,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if download_instructions is None: download_instructions = self.cfg['download_instructions'] if download_instructions is not None and download_instructions != "": - msg = "\nDownload instructions:\n\n" + download_instructions + '\n' + msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n' + msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths()) print_msg(msg, prefix=False, stderr=True) error_msg += "please follow the download instructions above, and make the file available " error_msg += "in the active source path (%s)" % ':'.join(source_paths()) From 3e8632f7ab414ecbc52ac59eee6bcfb31133cdcb Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 17 Feb 2024 13:09:40 +0000 Subject: [PATCH 076/171] update test --- test/framework/easyblock.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 64c554ca89..93d9fed793 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1582,7 +1582,8 @@ def test_download_instructions(self): self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) stderr = self.get_stderr().strip() self.mock_stderr(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '') @@ -1596,7 +1597,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # wipe top-level download instructions, try again self.contents = self.contents.replace(download_instructions, '') @@ -1625,7 +1627,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # download instructions should also be printed if 'source_tmpl' is used to specify extension sources self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,") @@ -1638,7 +1641,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file for extension write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') From b3092bfc9fec66ad7fa4d1be5b24ed2997bb4414 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:32:06 +0100 Subject: [PATCH 077/171] add unit tests for Extension.install_extension_substep --- test/framework/easyblock.py | 52 +++++++++++++++++++ .../generic/childcustomdummyextension.py | 36 +++++++++++++ .../generic/childdeprecateddummyextension.py | 36 +++++++++++++ .../generic/customdummyextension.py | 40 ++++++++++++++ .../generic/deprecateddummyextension.py | 40 ++++++++++++++ 5 files changed, 204 insertions(+) create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py create mode 100644 test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e29d900497..ccd72aa0e2 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1038,6 +1038,58 @@ def test_extensions_step(self): eb.close_log() os.remove(eb.logfile) + def test_extensions_step_deprecations(self): + """Test extension install with deprecated substeps.""" + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + 'exts_defaultclass = "DummyExtension"', + 'exts_list = ["ext1"]', + 'exts_list = [', + ' "dummy_ext",', + ' ("custom_ext", "0.0", {"easyblock": "CustomDummyExtension"}),', + ' ("deprec_ext", "0.0", {"easyblock": "DeprecatedDummyExtension"}),', + ' ("childcustom_ext", "0.0", {"easyblock": "ChildCustomDummyExtension"}),', + ' ("childdeprec_ext", "0.0", {"easyblock": "ChildDeprecatedDummyExtension"}),', + ']', + ]) + write_file(test_ec, test_ec_txt) + ec = process_easyconfig(test_ec)[0] + eb = get_easyblock_instance(ec) + eb.prepare_for_extensions() + eb.init_ext_instances() + + # Default DummyExtension without deprecated or custom install substeps + ext = eb.ext_instances[0] + self.assertEqual(ext.__class__.__name__, "DummyExtension") + self.assertEqual(ext.install_extension_substep("install_extension"), None) + # CustomDummyExtension + ext = eb.ext_instances[1] + self.assertEqual(ext.__class__.__name__, "CustomDummyExtension") + expected_return = "Extension installed with install_extension()" + self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + # DeprecatedDummyExtension + ext = eb.ext_instances[2] + self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension") + error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" + self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + # ChildCustomDummyExtension + ext = eb.ext_instances[3] + self.assertEqual(ext.__class__.__name__, "ChildCustomDummyExtension") + expected_return = "Extension installed with install_extension()" + self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + # ChildDeprecatedDummyExtension + ext = eb.ext_instances[4] + self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension") + error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" + self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + def test_init_extensions(self): """Test creating extension instances.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py new file mode 100644 index 0000000000..43b2ef6cbe --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py @@ -0,0 +1,36 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with customized methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.customdummyextension import CustomDummyExtension + + +class ChildCustomDummyExtension(CustomDummyExtension): + """Extension EasyBlock inheriting customized install step""" + diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py new file mode 100644 index 0000000000..327c64118d --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py @@ -0,0 +1,36 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with deprecated methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.deprecateddummyextension import DeprecatedDummyExtension + + +class ChildDeprecatedDummyExtension(DeprecatedDummyExtension): + """Extension EasyBlock inheriting deprecated install step""" + diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py new file mode 100644 index 0000000000..002d2e0008 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py @@ -0,0 +1,40 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with customized methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.dummyextension import DummyExtension + + +class CustomDummyExtension(DummyExtension): + """Extension EasyBlock with customized install step""" + + def install_extension(self): + + return "Extension installed with install_extension()" + diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py new file mode 100644 index 0000000000..0c3c397371 --- /dev/null +++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py @@ -0,0 +1,40 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Test EasyBlocks building and installing dummy extensions with deprecated methods + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.easyblocks.generic.dummyextension import DummyExtension + + +class DeprecatedDummyExtension(DummyExtension): + """Extension EasyBlock with deprecated install step""" + + def run(self): + + return "Extension installed with run()" + From 6a1b11315275cb3673c8e68363d3174e888bda06 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:33:35 +0100 Subject: [PATCH 078/171] fix code formatting in test easyblocks based on DummyExtension --- .../easybuild/easyblocks/generic/childcustomdummyextension.py | 1 - .../easyblocks/generic/childdeprecateddummyextension.py | 1 - .../sandbox/easybuild/easyblocks/generic/customdummyextension.py | 1 - .../easybuild/easyblocks/generic/deprecateddummyextension.py | 1 - 4 files changed, 4 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py index 43b2ef6cbe..4973f681e3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py @@ -33,4 +33,3 @@ class ChildCustomDummyExtension(CustomDummyExtension): """Extension EasyBlock inheriting customized install step""" - diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py index 327c64118d..e72b97940a 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py @@ -33,4 +33,3 @@ class ChildDeprecatedDummyExtension(DeprecatedDummyExtension): """Extension EasyBlock inheriting deprecated install step""" - diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py index 002d2e0008..da1144d980 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py @@ -37,4 +37,3 @@ class CustomDummyExtension(DummyExtension): def install_extension(self): return "Extension installed with install_extension()" - diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py index 0c3c397371..eb762da6e3 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py @@ -37,4 +37,3 @@ class DeprecatedDummyExtension(DummyExtension): def run(self): return "Extension installed with run()" - From d31d260eb9e862c8c06082a59f8bb9b697f37528 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:52:48 +0100 Subject: [PATCH 079/171] also test pre- and post- install extension subteps with Extension.install_extension_substep --- test/framework/easyblock.py | 24 ++++++++++++------- .../generic/customdummyextension.py | 10 +++++++- .../generic/deprecateddummyextension.py | 10 +++++++- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index ccd72aa0e2..e0eb06a823 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1040,6 +1040,7 @@ def test_extensions_step(self): def test_extensions_step_deprecations(self): """Test extension install with deprecated substeps.""" + install_substeps = ["pre_install_extension", "install_extension", "post_install_extension"] test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ @@ -1068,27 +1069,32 @@ def test_extensions_step_deprecations(self): # Default DummyExtension without deprecated or custom install substeps ext = eb.ext_instances[0] self.assertEqual(ext.__class__.__name__, "DummyExtension") - self.assertEqual(ext.install_extension_substep("install_extension"), None) + for substep in install_substeps: + self.assertEqual(ext.install_extension_substep(substep), None) # CustomDummyExtension ext = eb.ext_instances[1] self.assertEqual(ext.__class__.__name__, "CustomDummyExtension") - expected_return = "Extension installed with install_extension()" - self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + for substep in install_substeps: + expected_return = f"Extension installed with custom {substep}()" + self.assertEqual(ext.install_extension_substep(substep), expected_return) # DeprecatedDummyExtension ext = eb.ext_instances[2] self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension") - error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" - self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + for substep in install_substeps: + expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) # ChildCustomDummyExtension ext = eb.ext_instances[3] self.assertEqual(ext.__class__.__name__, "ChildCustomDummyExtension") - expected_return = "Extension installed with install_extension()" - self.assertEqual(ext.install_extension_substep("install_extension"), expected_return) + for substep in install_substeps: + expected_return = f"Extension installed with custom {substep}()" + self.assertEqual(ext.install_extension_substep(substep), expected_return) # ChildDeprecatedDummyExtension ext = eb.ext_instances[4] self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension") - error_msg = r"DEPRECATED \(since v6.0\).*use install_extension\(\) instead.*" - self.assertErrorRegex(EasyBuildError, error_msg, ext.install_extension_substep, "install_extension") + for substep in install_substeps: + expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) def test_init_extensions(self): """Test creating extension instances.""" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py index da1144d980..17a4434fe0 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py @@ -34,6 +34,14 @@ class CustomDummyExtension(DummyExtension): """Extension EasyBlock with customized install step""" + def pre_install_extension(self): + + return "Extension installed with custom pre_install_extension()" + def install_extension(self): - return "Extension installed with install_extension()" + return "Extension installed with custom install_extension()" + + def post_install_extension(self): + + return "Extension installed with custom post_install_extension()" diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py index eb762da6e3..46f10d49ae 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py @@ -34,6 +34,14 @@ class DeprecatedDummyExtension(DummyExtension): """Extension EasyBlock with deprecated install step""" + def prerun(self): + + return "Extension installed with custom prerun()" + def run(self): - return "Extension installed with run()" + return "Extension installed with custom run()" + + def postrun(self): + + return "Extension installed with custom postrun()" From fe7190a65f00ddd1f9eb7677f7b26c5284e41ab4 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 00:55:57 +0100 Subject: [PATCH 080/171] use raw string formatting in EasyBlockTest.test_extensions_step_deprecations --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e0eb06a823..74b294635f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1081,7 +1081,7 @@ def test_extensions_step_deprecations(self): ext = eb.ext_instances[2] self.assertEqual(ext.__class__.__name__, "DeprecatedDummyExtension") for substep in install_substeps: - expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) # ChildCustomDummyExtension ext = eb.ext_instances[3] @@ -1093,7 +1093,7 @@ def test_extensions_step_deprecations(self): ext = eb.ext_instances[4] self.assertEqual(ext.__class__.__name__, "ChildDeprecatedDummyExtension") for substep in install_substeps: - expected_error = f"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" + expected_error = rf"DEPRECATED \(since v6.0\).*use {substep}\(\) instead.*" self.assertErrorRegex(EasyBuildError, expected_error, ext.install_extension_substep, substep) def test_init_extensions(self): From 3b98a926e34bc8dff6a5a764c46a1fc191e6ec55 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 09:49:35 +0100 Subject: [PATCH 081/171] update docs tests with easyblocks based on DummyExtension --- test/framework/docs.py | 88 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 70280892e4..3fbe60c23f 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -59,6 +59,10 @@ |-- EB_toy_buggy |-- ExtensionEasyBlock | |-- DummyExtension +| | |-- CustomDummyExtension +| | | |-- ChildCustomDummyExtension +| | |-- DeprecatedDummyExtension +| | | |-- ChildDeprecatedDummyExtension | |-- EB_toy | | |-- EB_toy_eula | | |-- EB_toytoy @@ -69,6 +73,10 @@ Extension |-- ExtensionEasyBlock | |-- DummyExtension +| | |-- CustomDummyExtension +| | | |-- ChildCustomDummyExtension +| | |-- DeprecatedDummyExtension +| | | |-- ChildDeprecatedDummyExtension | |-- EB_toy | | |-- EB_toy_eula | | |-- EB_toytoy @@ -91,6 +99,10 @@ |-- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -101,6 +113,10 @@ Extension (easybuild.framework.extension) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -129,6 +145,16 @@ * ExtensionEasyBlock * DummyExtension + + * CustomDummyExtension + + * ChildCustomDummyExtension + + * DeprecatedDummyExtension + + * ChildDeprecatedDummyExtension + + * EB_toy * EB_toy_eula @@ -145,6 +171,16 @@ * ExtensionEasyBlock * DummyExtension + + * CustomDummyExtension + + * ChildCustomDummyExtension + + * DeprecatedDummyExtension + + * ChildDeprecatedDummyExtension + + * EB_toy * EB_toy_eula @@ -177,6 +213,16 @@ * ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + + * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) * EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) @@ -193,6 +239,16 @@ * ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + + * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) * EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) @@ -219,6 +275,10 @@ - EB_toy_buggy - ExtensionEasyBlock - DummyExtension + - CustomDummyExtension + - ChildCustomDummyExtension + - DeprecatedDummyExtension + - ChildDeprecatedDummyExtension - EB_toy - EB_toy_eula - EB_toytoy @@ -229,6 +289,10 @@ - **Extension** - ExtensionEasyBlock - DummyExtension + - CustomDummyExtension + - ChildCustomDummyExtension + - DeprecatedDummyExtension + - ChildDeprecatedDummyExtension - EB_toy - EB_toy_eula - EB_toytoy @@ -251,6 +315,10 @@ - EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -261,6 +329,10 @@ - **Extension** (easybuild.framework.extension) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -513,8 +585,20 @@ def test_get_easyblock_classes(self): # result should correspond with test easyblocks in test/framework/sandbox/easybuild/easyblocks/generic eb_classes = get_easyblock_classes('easybuild.easyblocks.generic') eb_names = [x.__name__ for x in eb_classes] - expected = ['ConfigureMake', 'DummyExtension', 'MakeCp', 'ModuleRC', - 'PythonBundle', 'Toolchain', 'Toy_Extension', 'bar'] + expected = [ + 'ChildCustomDummyExtension', + 'ChildDeprecatedDummyExtension', + 'ConfigureMake', + 'CustomDummyExtension', + 'DeprecatedDummyExtension', + 'DummyExtension', + 'MakeCp', + 'ModuleRC', + 'PythonBundle', + 'Toolchain', + 'Toy_Extension', + 'bar', + ] self.assertEqual(sorted(eb_names), expected) def test_gen_easyblocks_overview(self): From 02810a92b940b2654c2dcba0128bde549a37d052 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 10:16:17 +0100 Subject: [PATCH 082/171] replace hardcoded paths with topdir template --- test/framework/docs.py | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index 3fbe60c23f..bcb55b279d 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -99,10 +99,10 @@ |-- EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) -| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) -| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) -| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) -| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -113,10 +113,10 @@ Extension (easybuild.framework.extension) |-- ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) | |-- DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) -| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) -| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) -| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) -| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) +| | |-- CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) +| | | |-- ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) +| | |-- DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) +| | | |-- ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -214,13 +214,13 @@ * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) - * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) - * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) - * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) @@ -240,13 +240,13 @@ * DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) + * CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) - * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) + * ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) - * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) + * DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) - * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + * ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) * EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) @@ -315,10 +315,10 @@ - EB_toy_buggy (easybuild.easyblocks.toy_buggy @ %(topdir)s/t/toy_buggy.py) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) - - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) - - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) - - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) @@ -329,10 +329,10 @@ - **Extension** (easybuild.framework.extension) - ExtensionEasyBlock (easybuild.framework.extensioneasyblock ) - DummyExtension (easybuild.easyblocks.generic.dummyextension @ %(topdir)s/generic/dummyextension.py) - - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/customdummyextension.py) - - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childcustomdummyextension.py) - - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/deprecateddummyextension.py) - - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ /home/lexming/src/easybuild-framework/test/framework/sandbox/easybuild/easyblocks/generic/childdeprecateddummyextension.py) + - CustomDummyExtension (easybuild.easyblocks.generic.customdummyextension @ %(topdir)s/generic/customdummyextension.py) + - ChildCustomDummyExtension (easybuild.easyblocks.generic.childcustomdummyextension @ %(topdir)s/generic/childcustomdummyextension.py) + - DeprecatedDummyExtension (easybuild.easyblocks.generic.deprecateddummyextension @ %(topdir)s/generic/deprecateddummyextension.py) + - ChildDeprecatedDummyExtension (easybuild.easyblocks.generic.childdeprecateddummyextension @ %(topdir)s/generic/childdeprecateddummyextension.py) - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) From 1ed4efe7299a304813cbe75eb899e70a5fd47870 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 10:54:57 +0100 Subject: [PATCH 083/171] disable linter checks on text blocks in the docs tests --- test/framework/docs.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index bcb55b279d..ce6fea31fc 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -80,7 +80,7 @@ | |-- EB_toy | | |-- EB_toy_eula | | |-- EB_toytoy -| |-- Toy_Extension""" +| |-- Toy_Extension""" # noqa LIST_EASYBLOCKS_DETAILED_TXT = """EasyBlock (easybuild.framework.easyblock) |-- bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) @@ -120,7 +120,7 @@ | |-- EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) | | |-- EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) | | |-- EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) -| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" +| |-- Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa LIST_EASYBLOCKS_SIMPLE_RST = """* **EasyBlock** @@ -188,7 +188,7 @@ * Toy_Extension -""" +""" # noqa LIST_EASYBLOCKS_DETAILED_RST = """* **EasyBlock** (easybuild.framework.easyblock) @@ -256,7 +256,7 @@ * Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py) -""" +""" # noqa LIST_EASYBLOCKS_SIMPLE_MD = """- **EasyBlock** - bar @@ -296,7 +296,7 @@ - EB_toy - EB_toy_eula - EB_toytoy - - Toy_Extension""" + - Toy_Extension""" # noqa LIST_EASYBLOCKS_DETAILED_MD = """- **EasyBlock** (easybuild.framework.easyblock) - bar (easybuild.easyblocks.generic.bar @ %(topdir)s/generic/bar.py) @@ -336,11 +336,11 @@ - EB_toy (easybuild.easyblocks.toy @ %(topdir)s/t/toy.py) - EB_toy_eula (easybuild.easyblocks.toy_eula @ %(topdir)s/t/toy_eula.py) - EB_toytoy (easybuild.easyblocks.toytoy @ %(topdir)s/t/toytoy.py) - - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" + - Toy_Extension (easybuild.easyblocks.generic.toy_extension @ %(topdir)s/generic/toy_extension.py)""" # noqa LIST_SOFTWARE_SIMPLE_TXT = """ * GCC -* gzip""" +* gzip""" # noqa GCC_DESCR = "The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, " GCC_DESCR += "as well as libraries for these languages (libstdc++, libgcj,...)." @@ -363,7 +363,7 @@ * gzip v1.4: GCC/4.6.3, system * gzip v1.5: foss/2018a, intel/2018a -""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_RST = """List of supported software ========================== @@ -379,7 +379,7 @@ --- * GCC -* gzip""" +* gzip""" # noqa LIST_SOFTWARE_DETAILED_RST = """List of supported software ========================== @@ -429,7 +429,7 @@ ``1.4`` ``GCC/4.6.3``, ``system`` ``1.5`` ``foss/2018a``, ``intel/2018a`` ======= =============================== -""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_MD = """# List of supported software @@ -441,7 +441,7 @@ ## G * GCC -* gzip""" +* gzip""" # noqa LIST_SOFTWARE_DETAILED_MD = """# List of supported software @@ -475,7 +475,7 @@ version|toolchain -------|------------------------------- ``1.4``|``GCC/4.6.3``, ``system`` -``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_MD = """# List of supported software @@ -487,7 +487,7 @@ ## G * GCC -* gzip""" +* gzip""" # noqa LIST_SOFTWARE_DETAILED_MD = """# List of supported software @@ -521,7 +521,7 @@ version|toolchain -------|------------------------------- ``1.4``|``GCC/4.6.3``, ``system`` -``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa LIST_SOFTWARE_SIMPLE_JSON = """[ { @@ -530,7 +530,7 @@ { "name": "gzip" } -]""" +]""" # noqa LIST_SOFTWARE_DETAILED_JSON = """[ { @@ -573,7 +573,7 @@ "version": "1.5", "versionsuffix": "" } -]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} # noqa class DocsTest(EnhancedTestCase): From dd2da2139aa601f857ba2f8047931982a555a6b4 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 19 Feb 2024 11:12:54 +0100 Subject: [PATCH 084/171] update options tests with easyblocks based on DummyExtension --- test/framework/options.py | 71 ++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index ca8a582365..820e4a9c56 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -923,37 +923,46 @@ def test_000_list_easyblocks(self): logtxt = read_file(self.logfile) expected = '\n'.join([ - r'EasyBlock', - r'\|-- bar', - r'\|-- ConfigureMake', - r'\| \|-- MakeCp', - r'\|-- EB_EasyBuildMeta', - r'\|-- EB_FFTW', - r'\|-- EB_foo', - r'\| \|-- EB_foofoo', - r'\|-- EB_GCC', - r'\|-- EB_HPL', - r'\|-- EB_libtoy', - r'\|-- EB_OpenBLAS', - r'\|-- EB_OpenMPI', - r'\|-- EB_ScaLAPACK', - r'\|-- EB_toy_buggy', - r'\|-- ExtensionEasyBlock', - r'\| \|-- DummyExtension', - r'\| \|-- EB_toy', - r'\| \| \|-- EB_toy_eula', - r'\| \| \|-- EB_toytoy', - r'\| \|-- Toy_Extension', - r'\|-- ModuleRC', - r'\|-- PythonBundle', - r'\|-- Toolchain', - r'Extension', - r'\|-- ExtensionEasyBlock', - r'\| \|-- DummyExtension', - r'\| \|-- EB_toy', - r'\| \| \|-- EB_toy_eula', - r'\| \| \|-- EB_toytoy', - r'\| \|-- Toy_Extension', + "EasyBlock", + "|-- bar", + "|-- ConfigureMake", + "| |-- MakeCp", + "|-- EB_EasyBuildMeta", + "|-- EB_FFTW", + "|-- EB_foo", + "| |-- EB_foofoo", + "|-- EB_GCC", + "|-- EB_HPL", + "|-- EB_libtoy", + "|-- EB_OpenBLAS", + "|-- EB_OpenMPI", + "|-- EB_ScaLAPACK", + "|-- EB_toy_buggy", + "|-- ExtensionEasyBlock", + "| |-- DummyExtension", + "| | |-- CustomDummyExtension", + "| | | |-- ChildCustomDummyExtension", + "| | |-- DeprecatedDummyExtension", + "| | | |-- ChildDeprecatedDummyExtension", + "| |-- EB_toy", + "| | |-- EB_toy_eula", + "| | |-- EB_toytoy", + "| |-- Toy_Extension", + "|-- ModuleRC", + "|-- PythonBundle", + "|-- Toolchain", + "Extension", + "|-- ExtensionEasyBlock", + "| |-- DummyExtension", + "| | |-- CustomDummyExtension", + "| | | |-- ChildCustomDummyExtension", + "| | |-- DeprecatedDummyExtension", + "| | | |-- ChildDeprecatedDummyExtension", + "| |-- EB_toy", + "| | |-- EB_toy_eula", + "| | |-- EB_toytoy", + "| |-- Toy_Extension", + "", ]) regex = re.compile(expected, re.M) self.assertTrue(regex.search(logtxt), "Pattern '%s' found in: %s" % (regex.pattern, logtxt)) From ca3828bc1c1a16f29287e62b11c6a705aa86aceb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 14:34:09 +0100 Subject: [PATCH 085/171] refactor EasyBlock.skip_extensions_parallel --- easybuild/framework/easyblock.py | 33 ++++++++++---------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index e2d84c162e..64aa182dc7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -41,6 +41,7 @@ * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) """ +import concurrent import copy import glob import inspect @@ -1818,22 +1819,20 @@ def skip_extensions_parallel(self, exts_filter): self.log.experimental("Skipping installed extensions in parallel") print_msg("skipping installed extensions (in parallel)", log=self.log) - thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) - - async_shell_cmd_tasks = [] installed_exts_ids = [] - exts_queue = list(enumerate(self.ext_instances[:])) checked_exts_cnt = 0 exts_cnt = len(self.ext_instances) - done_tasks = [] + cmds = [resolve_exts_filter_template(exts_filter, ext) for ext in self.ext_instances] - # asynchronously run checks to see whether extensions are already installed - while exts_queue or async_shell_cmd_tasks: + with ThreadPoolExecutor(max_workers=self.cfg['parallel']) as thread_pool: - # first handle completed checks - for task in done_tasks: - async_shell_cmd_tasks.remove(task) - res = task.result() + # list of command to run asynchronously + async_cmds = [thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, fail_on_error=False, + asynchronous=True, task_id=idx) for (idx, (cmd, stdin)) in enumerate(cmds)] + + # process result of commands as they have completed running + for done_task in concurrent.futures.as_completed(async_cmds): + res = done_task.result() idx = res.task_id ext_name = self.ext_instances[idx].name self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") @@ -1846,16 +1845,6 @@ def skip_extensions_parallel(self, exts_filter): exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) self.update_exts_progress_bar(exts_pbar_label) - # start additional checks asynchronously - while exts_queue: - idx, ext = exts_queue.pop(0) - cmd, stdin = resolve_exts_filter_template(exts_filter, ext) - task = thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, - fail_on_error=False, asynchronous=True, task_id=idx) - async_shell_cmd_tasks.append(task) - - (done_tasks, _) = wait(async_shell_cmd_tasks, timeout=1, return_when=FIRST_COMPLETED) - # compose new list of extensions, skip over the ones that are already installed; # note: original order in extensions list should be preserved! retained_ext_instances = [] @@ -1866,8 +1855,6 @@ def skip_extensions_parallel(self, exts_filter): self.ext_instances = retained_ext_instances - thread_pool.shutdown() - def install_extensions(self, install=True): """ Install extensions. From f7c0ff2295789736557861ede1b82172e40f6ab7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 15:51:16 +0100 Subject: [PATCH 086/171] clean up unused imports from concurrent.futures --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 64aa182dc7..4dd4bcfd81 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -52,7 +52,7 @@ import tempfile import time import traceback -from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from concurrent.futures import ThreadPoolExecutor from datetime import datetime import easybuild.tools.environment as env From 4ff3663e6dbb9aa18a875acd00784559df2352cb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 13 Feb 2024 21:26:43 +0100 Subject: [PATCH 087/171] implement basic support for qa_patterns in run_shell_cmd --- easybuild/tools/run.py | 36 +++++++++++++++++++++++++++++++----- test/framework/run.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 95391f3821..da259a8721 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -36,6 +36,7 @@ * Ward Poelmans (Ghent University) """ import contextlib +import fcntl import functools import inspect import os @@ -231,7 +232,7 @@ def to_cmd_str(cmd): return cmd_str # temporarily raise a NotImplementedError until all options are implemented - if qa_patterns or qa_wait_patterns: + if qa_wait_patterns: raise NotImplementedError if work_dir is None: @@ -315,23 +316,48 @@ def to_cmd_str(cmd): if stdin: stdin = stdin.encode() - if stream_output: + if stream_output or qa_patterns: + + if qa_patterns: + # make stdout, stderr, stdin non-blocking files + channels = [proc.stdout, proc.stdin] + if split_stderr: + channels += proc.stderr + for channel in channels: + fd = channel.fileno() + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + if stdin: proc.stdin.write(stdin) exit_code = None stdout, stderr = b'', b'' + # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided) while exit_code is None: - exit_code = proc.poll() # use small read size (128 bytes) when streaming output, to make it stream more fluently # -1 means reading until EOF read_size = 128 if exit_code is None else -1 - stdout += proc.stdout.read(read_size) + exit_code = proc.poll() + + more_stdout = proc.stdout.read1(read_size) or b'' + stdout += more_stdout + + # note: we assume that there won't be any questions in stderr output if split_stderr: - stderr += proc.stderr.read(read_size) + stderr += proc.stderr.read1(read_size) or b'' + + # only consider answering questions if there's new output beyond additional whitespace + if more_stdout.strip() and qa_patterns: + for question, answer in qa_patterns: + question += '[\s\n]*$' + regex = re.compile(question.encode()) + if regex.search(stdout): + answer += '\n' + x= os.write(proc.stdin.fileno(), answer.encode()) else: (stdout, stderr) = proc.communicate(input=stdin) diff --git a/test/framework/run.py b/test/framework/run.py index c86d8635ed..887a9b0030 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -750,6 +750,40 @@ def test_run_cmd_qa(self): self.assertTrue(out.startswith("question\nanswer\nfoo ")) self.assertTrue(out.endswith('bar')) + def test_run_shell_cmd_qa(self): + """Basic test for Q&A support in run_shell_cmd function.""" + + cmd = '; '.join([ + "echo question1", + "read x", + "echo $x", + "echo question2", + "read y", + "echo $y", + ]) + qa = [ + ('question1', 'answer1'), + ('question2', 'answer2'), + ] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.output, "question1\nanswer1\nquestion2\nanswer2\n") + # no reason echo hello could fail + self.assertEqual(res.exit_code, 0) + + # test running command that emits non-UTF8 characters + # this is constructed to reproduce errors like: + # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 + test_file = os.path.join(self.test_prefix, 'foo.txt') + write_file(test_file, b"foo \xe2 bar") + cmd += "; cat %s" % test_file + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.exit_code, 0) + self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo ")) + self.assertTrue(res.output.endswith('bar')) + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" From 1cce87361f6465f7524b10e5d3ac9887e93dae03 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 20:51:12 +0100 Subject: [PATCH 088/171] add support for qa_timeout in run_shell_cmd + check type of qa_patterns value --- easybuild/tools/run.py | 23 ++++++++++++++++++++- test/framework/run.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index da259a8721..538c952cfb 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -192,7 +192,7 @@ def cache_aware_func(cmd, *args, **kwargs): def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, - qa_patterns=None, qa_wait_patterns=None): + qa_patterns=None, qa_wait_patterns=None, qa_timeout=10000): """ Run specified (interactive) shell command, and capture output + exit code. @@ -213,6 +213,8 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions and number of iterations to allow these patterns to match with end out command output + :param qa_timeout: amount of milliseconds to wait until more output is produced when there is no matching question + :return: Named tuple with: - output: command output, stdout+stderr combined if split_stderr is disabled, only stdout otherwise - exit_code: exit code of command (integer) @@ -231,6 +233,11 @@ def to_cmd_str(cmd): return cmd_str + # make sure that qa_patterns is a list of 2-tuples (not a dict, or something else) + if qa_patterns: + if not isinstance(qa_patterns, list) or any(not isinstance(x, tuple) or len(x) != 2 for x in qa_patterns): + raise EasyBuildError("qa_patterns passed to run_shell_cmd should be a list of 2-tuples!") + # temporarily raise a NotImplementedError until all options are implemented if qa_wait_patterns: raise NotImplementedError @@ -333,6 +340,8 @@ def to_cmd_str(cmd): exit_code = None stdout, stderr = b'', b'' + check_interval_secs = 0.001 + time_no_match = 0 # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided) while exit_code is None: @@ -351,6 +360,7 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' # only consider answering questions if there's new output beyond additional whitespace + hit = False if more_stdout.strip() and qa_patterns: for question, answer in qa_patterns: question += '[\s\n]*$' @@ -358,6 +368,17 @@ def to_cmd_str(cmd): if regex.search(stdout): answer += '\n' x= os.write(proc.stdin.fileno(), answer.encode()) + hit = True + + time.sleep(check_interval_secs) + if hit: + time_no_match = 0 + else: + time_no_match += check_interval_secs + if time_no_match >= qa_timeout: + error_msg = "No matching questions found for current command output, " + error_msg += f"giving up after {qa_timeout} seconds!" + raise EasyBuildError(error_msg) else: (stdout, stderr) = proc.communicate(input=stdin) diff --git a/test/framework/run.py b/test/framework/run.py index 887a9b0030..0dd1b4888b 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -784,6 +784,21 @@ def test_run_shell_cmd_qa(self): self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo ")) self.assertTrue(res.output.endswith('bar')) + # check type check on qa_patterns + error_pattern = "qa_patterns passed to run_shell_cmd should be a list of 2-tuples!" + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns={'foo': 'bar'}) + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=('foo', 'bar')) + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=(('foo', 'bar'),)) + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns='foo:bar') + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=['foo:bar']) + + # validate use of qa_timeout to give up if there's no matching question for too long + cmd = "sleep 3; echo 'question'; read a; echo $a" + error_pattern = "No matching questions found for current command output, giving up after 1 seconds!" + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1) + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" @@ -816,6 +831,38 @@ def test_run_cmd_qa_buffering(self): self.assertEqual(ec, 1) self.assertEqual(out, "Hello, I am about to exit\nERROR: I failed\n") + def test_run_shell_cmd_qa_buffering(self): + """Test whether run_shell_cmd uses unbuffered output when running interactive commands.""" + + # command that generates a lot of output before waiting for input + # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000! + cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; ' + cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=[('Pick a number: ', '42')], qa_timeout=10) + + self.assertEqual(res.exit_code, 0) + regex = re.compile("Picked number: 42$") + self.assertTrue(regex.search(res.output), f"Pattern '{regex.pattern}' found in: {res.output}") + + # also test with script run as interactive command that quickly exits with non-zero exit code; + # see https://github.com/easybuilders/easybuild-framework/issues/3593 + script_txt = '\n'.join([ + "#/bin/bash", + "echo 'Hello, I am about to exit'", + "echo 'ERROR: I failed' >&2", + "exit 1", + ]) + script = os.path.join(self.test_prefix, 'test.sh') + write_file(script, script_txt) + adjust_permissions(script, stat.S_IXUSR) + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(script, qa_patterns=[], fail_on_error=False) + + self.assertEqual(res.exit_code, 1) + self.assertEqual(res.output, "Hello, I am about to exit\nERROR: I failed\n") + def test_run_cmd_qa_log_all(self): """Test run_cmd_qa with log_output enabled""" with self.mocked_stdout_stderr(): From 651da9715f3f30a038a56e4d940ec5d21a3d0928 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 20:54:34 +0100 Subject: [PATCH 089/171] fix trivial code style issues --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 538c952cfb..78aeab85d8 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -363,11 +363,11 @@ def to_cmd_str(cmd): hit = False if more_stdout.strip() and qa_patterns: for question, answer in qa_patterns: - question += '[\s\n]*$' + question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): answer += '\n' - x= os.write(proc.stdin.fileno(), answer.encode()) + os.write(proc.stdin.fileno(), answer.encode()) hit = True time.sleep(check_interval_secs) diff --git a/test/framework/run.py b/test/framework/run.py index 0dd1b4888b..5697f39e91 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -779,7 +779,7 @@ def test_run_shell_cmd_qa(self): cmd += "; cat %s" % test_file with self.mocked_stdout_stderr(): - res = run_shell_cmd(cmd, qa_patterns=qa) + res = run_shell_cmd(cmd, qa_patterns=qa) self.assertEqual(res.exit_code, 0) self.assertTrue(res.output.startswith("question1\nanswer1\nquestion2\nanswer2\nfoo ")) self.assertTrue(res.output.endswith('bar')) From f01ef65d3cdb0bfa3a3fd80a33bbe874095dd512 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 21:01:54 +0100 Subject: [PATCH 090/171] simplify qa_timeout code in run_shell_cmd by using for-else statement --- easybuild/tools/run.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 78aeab85d8..93e4afa65d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -360,25 +360,24 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' # only consider answering questions if there's new output beyond additional whitespace - hit = False - if more_stdout.strip() and qa_patterns: + if qa_patterns: for question, answer in qa_patterns: question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) - hit = True + time_no_match = 0 + break + else: + # this will only run if the for loop above was *not* stopped by the break statement + time_no_match += check_interval_secs + if time_no_match > qa_timeout: + error_msg = "No matching questions found for current command output, " + error_msg += f"giving up after {qa_timeout} seconds!" + raise EasyBuildError(error_msg) time.sleep(check_interval_secs) - if hit: - time_no_match = 0 - else: - time_no_match += check_interval_secs - if time_no_match >= qa_timeout: - error_msg = "No matching questions found for current command output, " - error_msg += f"giving up after {qa_timeout} seconds!" - raise EasyBuildError(error_msg) else: (stdout, stderr) = proc.communicate(input=stdin) From 5899ce284ef45dce7c648ace17a2b95487e4765a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 21:24:29 +0100 Subject: [PATCH 091/171] fix default value for qa_timeout (we're counting seconds, not milliseconds) --- easybuild/tools/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 93e4afa65d..1ffcd9054d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -192,7 +192,7 @@ def cache_aware_func(cmd, *args, **kwargs): def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, - qa_patterns=None, qa_wait_patterns=None, qa_timeout=10000): + qa_patterns=None, qa_wait_patterns=None, qa_timeout=100): """ Run specified (interactive) shell command, and capture output + exit code. @@ -213,7 +213,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions and number of iterations to allow these patterns to match with end out command output - :param qa_timeout: amount of milliseconds to wait until more output is produced when there is no matching question + :param qa_timeout: amount of seconds to wait until more output is produced when there is no matching question :return: Named tuple with: - output: command output, stdout+stderr combined if split_stderr is disabled, only stdout otherwise From cfb7f3ad9edd9cdbc7f8515a5d5157cdfe519257 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Feb 2024 21:39:04 +0100 Subject: [PATCH 092/171] fix trace message for interactive commands run with run_shell_cmd --- easybuild/tools/run.py | 14 +++++++----- test/framework/run.py | 48 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 1ffcd9054d..5a8bc298ff 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -277,7 +277,8 @@ def to_cmd_str(cmd): if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: silent = build_option('silent') - msg = f" running shell command \"{cmd_str}\"\n" + interactive = 'interactive ' if qa_patterns else '' + msg = f" running {interactive}shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) @@ -286,7 +287,8 @@ def to_cmd_str(cmd): start_time = datetime.now() if not hidden: - _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, + interactive=bool(qa_patterns)) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -430,7 +432,7 @@ def to_cmd_str(cmd): return res -def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id): +def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, interactive=False): """ Helper function to construct and print trace message for command being run @@ -441,13 +443,15 @@ def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thr :param cmd_out_fp: path to output file for command :param cmd_err_fp: path to errors/warnings output file for command :param thread_id: thread ID (None when not running shell command asynchronously) + :param interactive: boolean indicating whether it is an interactive command, or not """ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + interactive = 'interactive ' if interactive else '' if thread_id: - run_cmd_msg = f"running shell command (asynchronously, thread ID: {thread_id}):" + run_cmd_msg = f"running {interactive}shell command (asynchronously, thread ID: {thread_id}):" else: - run_cmd_msg = "running shell command:" + run_cmd_msg = f"running {interactive}shell command:" lines = [ run_cmd_msg, diff --git a/test/framework/run.py b/test/framework/run.py index 5697f39e91..669b8fa073 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -876,13 +876,22 @@ def test_run_cmd_qa_log_all(self): extra_pref = "# output for interactive command: echo 'n: '; read n; seq 1 $n\n\n" self.assertEqual(run_cmd_log_txt, extra_pref + "n: \n1\n2\n3\n4\n5\n") + def test_run_shell_cmd_qa_log(self): + """Test temporary log file for run_shell_cmd with qa_patterns""" + with self.mocked_stdout_stderr(): + res = run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n:', '5')]) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "n: \n1\n2\n3\n4\n5\n") + + run_cmd_logs = glob.glob(os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output', 'echo-*', 'out.txt')) + self.assertEqual(len(run_cmd_logs), 1) + run_cmd_log_txt = read_file(run_cmd_logs[0]) + self.assertEqual(run_cmd_log_txt, "n: \n1\n2\n3\n4\n5\n") + def test_run_cmd_qa_trace(self): """Test run_cmd under --trace""" - # replace log.experimental with log.warning to allow experimental code - easybuild.tools.utilities._log.experimental = easybuild.tools.utilities._log.warning - - init_config(build_options={'trace': True}) + # --trace is enabled by default self.mock_stdout(True) self.mock_stderr(True) (out, ec) = run_cmd_qa("echo 'n: '; read n; seq 1 $n", {'n: ': '5'}) @@ -910,6 +919,37 @@ def test_run_cmd_qa_trace(self): self.assertEqual(stdout, '') self.assertEqual(stderr, '') + def test_run_shell_cmd_qa_trace(self): + """Test run_shell_cmd with qa_patterns under --trace""" + + # --trace is enabled by default + self.mock_stdout(True) + self.mock_stderr(True) + run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')]) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(stderr, '') + pattern = r"^ >> running interactive shell command:\n" + pattern += r"\techo \'n: \'; read n; seq 1 \$n\n" + pattern += r"\t\[started at: .*\]\n" + pattern += r"\t\[working dir: .*\]\n" + pattern += r"\t\[output saved to .*\]\n" + pattern += r' >> command completed: exit 0, ran in .*' + self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout)) + + # trace output can be disabled on a per-command basis + self.mock_stdout(True) + self.mock_stderr(True) + run_shell_cmd("echo 'n: '; read n; seq 1 $n", qa_patterns=[('n: ', '5')], hidden=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(stdout, '') + self.assertEqual(stderr, '') + def test_run_cmd_qa_answers(self): """Test providing list of answers in run_cmd_qa.""" cmd = "echo question; read x; echo $x; " * 2 From 4b63b913ef88b402036757dc2524b91f3b866bb4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Feb 2024 16:46:51 +0100 Subject: [PATCH 093/171] make sure that *all* stdout/stderr output is read when streaming output or running interactive commands --- easybuild/tools/run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 5a8bc298ff..09a699ca80 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -352,8 +352,6 @@ def to_cmd_str(cmd): # -1 means reading until EOF read_size = 128 if exit_code is None else -1 - exit_code = proc.poll() - more_stdout = proc.stdout.read1(read_size) or b'' stdout += more_stdout @@ -380,6 +378,12 @@ def to_cmd_str(cmd): raise EasyBuildError(error_msg) time.sleep(check_interval_secs) + + exit_code = proc.poll() + + stdout += proc.stdout.read() + if split_stderr: + stderr += proc.stderr.read() else: (stdout, stderr) = proc.communicate(input=stdin) From cb84bd2ba1610419e03f869053d3bd158e5ada1a Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:08:58 +0000 Subject: [PATCH 094/171] extra cases --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index a2854ad8fb..6a568a9f70 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -186,10 +186,10 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): Simplify the cmd to only the allowed_chars we want in a filename :param cmd: the cmd (string) - :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits) + :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits + "_-") """ if not allowed_chars: - allowed_chars = string.ascii_letters + string.digits + allowed_chars = f"{string.ascii_letters}{string.digits}_-" return ''.join([c for c in cmd if c in allowed_chars]) diff --git a/test/framework/run.py b/test/framework/run.py index 22fd2a86ba..386a9cf7f6 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -200,6 +200,8 @@ def test_fileprefix_from_cmd(self): 'abd123': 'abd123', 'ab"a': 'aba', 'a{:$:S@"a': 'aSa', + 'cmd-with-dash': 'cmd-with-dash', + 'cmd_with_underscore'. 'cmd_with_underscore', } for cmd, expected_simplification in cmds.items(): self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) From a309aaea65a335abee90f4a8df09907c92eaed20 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:09:40 +0000 Subject: [PATCH 095/171] typo --- test/framework/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index 386a9cf7f6..e63e4e219c 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -201,7 +201,7 @@ def test_fileprefix_from_cmd(self): 'ab"a': 'aba', 'a{:$:S@"a': 'aSa', 'cmd-with-dash': 'cmd-with-dash', - 'cmd_with_underscore'. 'cmd_with_underscore', + 'cmd_with_underscore': 'cmd_with_underscore', } for cmd, expected_simplification in cmds.items(): self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) From d57c251c8145ff080529038f00505951eaee6983 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:34:10 +0000 Subject: [PATCH 096/171] replace `'` with `"` for `printf` to have bash replace a variable --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ec98f9df03..c79b515e42 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -178,7 +178,7 @@ jobs: echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool" continue fi - printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n' + printf "\n\n=====================> Using $module_syntax module syntax <=====================\n\n" export EASYBUILD_MODULE_SYNTAX="${module_syntax}" export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}" From 560d395a1d8ab4ef2366543b642d15ec548c1c1a Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 16:13:41 +0000 Subject: [PATCH 097/171] re-order test_cuda_compute_capabilities {pre,}{config,build,install}opts --- test/framework/easyconfig.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 11b551e55c..1ce8d1facc 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4438,31 +4438,31 @@ def test_cuda_compute_capabilities(self): description = 'test' toolchain = SYSTEM cuda_compute_capabilities = ['5.1', '7.0', '7.1'] - installopts = '%(cuda_compute_capabilities)s' - preinstallopts = '%(cuda_cc_space_sep)s' - prebuildopts = '%(cuda_cc_semicolon_sep)s' - configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' + configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' + prebuildopts = '%(cuda_cc_semicolon_sep)s' + preinstallopts = '%(cuda_cc_space_sep)s' + installopts = '%(cuda_compute_capabilities)s' """) self.prep() ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '5.1,7.0,7.1') - self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') - self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') + self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['installopts'], '5.1,7.0,7.1') # build options overwrite it init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '4.2,6.3') - self.assertEqual(ec['preinstallopts'], '4.2 6.3') - self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') + self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['installopts'], '4.2,6.3') def test_det_copy_ec_specs(self): """Test det_copy_ec_specs function.""" From aff74821c1cc9b3e69065d370e4bd64318162a26 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 16:37:45 +0000 Subject: [PATCH 098/171] add support for cuda compute capabilities templates in the form of: cuda_int_comma_sep: 70,75 80 cuda_int_semicolon_sep: 70;75;80 cuda_int_space_sep: 70 75 80 --- easybuild/framework/easyconfig/templates.py | 7 +++++++ test/framework/easyconfig.py | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index fcefe4de55..0d8a4d5cf3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -97,6 +97,9 @@ ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), + ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"), + ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"), + ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), ] @@ -365,6 +368,10 @@ def template_constant_dict(config, ignore=None, toolchain=None): template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) + int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities] + template_values['cuda_int_comma_sep'] = ','.join(int_values) + template_values['cuda_int_space_sep'] = ' '.join(int_values) + template_values['cuda_int_semicolon_sep'] = ';'.join(int_values) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 1ce8d1facc..399ce3afe8 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,6 +4441,8 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" + 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' """) @@ -4451,6 +4453,9 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['buildopts'], 'comma="51,70,71" ' + 'space="51 70 71" ' + 'semi="51;70;71"') self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') self.assertEqual(ec['installopts'], '5.1,7.0,7.1') @@ -4460,6 +4465,9 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') + self.assertEqual(ec['buildopts'], 'comma="42,63" ' + 'space="42 63" ' + 'semi="42;63"') self.assertEqual(ec['prebuildopts'], '4.2;6.3') self.assertEqual(ec['preinstallopts'], '4.2 6.3') self.assertEqual(ec['installopts'], '4.2,6.3') @@ -4725,6 +4733,9 @@ def test_get_cuda_cc_template_value(self): 'cuda_compute_capabilities': '6.5,7.0', 'cuda_cc_space_sep': '6.5 7.0', 'cuda_cc_semicolon_sep': '6.5;7.0', + 'cuda_int_comma_sep': '65,70', + 'cuda_int_space_sep': '65 70', + 'cuda_int_semicolon_sep': '65;70', 'cuda_sm_comma_sep': 'sm_65,sm_70', 'cuda_sm_space_sep': 'sm_65 sm_70', } From f2d53d1ca449c404a246fe54bc9be897758ef39f Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 17:03:12 +0000 Subject: [PATCH 099/171] fix missing end quote --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 399ce3afe8..a61be39473 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,7 +4441,7 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' - buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s"' 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' From b65771f3132a0dc3b9a9fae6f3b015215637ba62 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 22 Feb 2024 17:41:50 +0000 Subject: [PATCH 100/171] fix missing space --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a61be39473..141e9b0f77 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -4441,7 +4441,7 @@ def test_cuda_compute_capabilities(self): preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' - buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s"' + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" ' 'semi="%(cuda_int_semicolon_sep)s"') preinstallopts = '%(cuda_cc_space_sep)s' installopts = '%(cuda_compute_capabilities)s' From 3a7b3eb109c8933c68ef2a89f71391b1c243e6a7 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 23 Feb 2024 16:38:36 +0100 Subject: [PATCH 101/171] Use cp -dR instead of cp -a for shell script extraction cp -a will try to copy attributes, when the copy is from a file system type with attributes that doesn't work on the target file system this will fail --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5a2f724bc8..bc938f4210 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -161,7 +161,7 @@ # tar.Z: using compress (LZW), but can be handled with gzip so use 'z' '.tar.z': "tar xzf %(filepath)s", # shell scripts don't need to be unpacked, just copy there - '.sh': "cp -a %(filepath)s .", + '.sh': "cp -dR %(filepath)s .", } ZIPPED_PATCH_EXTS = ('.bz2', '.gz', '.xz') From 17f4b1dff887fc69e0fec73c8919335878ec4edf Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 26 Feb 2024 09:36:19 +0800 Subject: [PATCH 102/171] fix link to documentation in close_pr message --- easybuild/tools/github.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8100c8613b..96ed4d6492 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1370,8 +1370,7 @@ def close_pr(pr, motivation_msg=None): if not reopen: msg += "\nPlease don't hesitate to reopen this PR or add a comment if you feel this contribution is still " msg += "relevant.\nFor more information on our policy w.r.t. closing PRs, see " - msg += "https://easybuild.readthedocs.io/en/latest/Contributing.html" - msg += "#why-a-pull-request-may-be-closed-by-a-maintainer" + msg += "https://docs.easybuild.io/contributing/#contributing_review_process_why_pr_closed_by_maintainer" post_comment_in_issue(pr, msg, account=pr_target_account, repo=pr_target_repo, github_user=github_user) if dry_run: From 0f18f73c8207f8172efa7dda07aed9aaadd4b070 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 26 Feb 2024 07:45:37 +0100 Subject: [PATCH 103/171] tests: Adjust test of EXTRACT_CMDS for .sh files to match the change from cp -a to cp -dR --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3b879bef85..8ee6b2c917 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -104,7 +104,7 @@ def test_extract_cmd(self): ('test.txz', "unset TAPE; unxz test.txz --stdout | tar x"), ('test.iso', "7z x test.iso"), ('test.tar.Z', "tar xzf test.tar.Z"), - ('test.foo.bar.sh', "cp -a test.foo.bar.sh ."), + ('test.foo.bar.sh', "cp -dR test.foo.bar.sh ."), # check whether extension is stripped correct to determine name of target file # cfr. https://github.com/easybuilders/easybuild-framework/pull/3705 ('testbz2.bz2', "bunzip2 -c testbz2.bz2 > testbz2"), From 8d9e14eb6d72490c675adb8a70415a4c735cc0bc Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 02:40:57 +0100 Subject: [PATCH 104/171] make reproducible archives only of git repos without .git dir and reset timestamps with touch --- easybuild/tools/filetools.py | 42 ++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 12fe557a60..5b7406fffa 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2540,12 +2540,12 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path) -def get_source_tarball_from_git(filename, targetdir, git_config): +def get_source_tarball_from_git(filename, target_dir, git_config): """ Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it :param filename: name of the archive to save the code to (must be .tar.gz) - :param targetdir: target directory where to save the archive to + :param target_dir: target directory where to save the archive to :param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit """ # sanity check on git_config value being passed @@ -2584,8 +2584,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): raise EasyBuildError("git_config currently only supports filename ending in .tar.gz") # prepare target directory and clone repository - mkdir(targetdir, parents=True) - targetpath = os.path.join(targetdir, filename) + mkdir(target_dir, parents=True) # compose 'git clone' command, and run it if extra_config_params: @@ -2668,21 +2667,36 @@ def get_source_tarball_from_git(filename, targetdir, git_config): for cmd in cmds: run_shell_cmd(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True) - # When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag - # '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and - # --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/ - tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|', - 'LC_ALL=C', 'sort', '--zero-terminated', '|', - 'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion', - '--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0', - '--numeric-owner', '--format=gnu', '--null', - '--no-recursion', '--files-from -'] + # Create archive + archive_path = os.path.join(target_dir, filename) + + if keep_git_dir: + # create archive of git repo including .git directory + tar_cmd = ['tar', 'cfvz', archive_path, repo_name] + else: + # create reproducible archive + # see https://reproducible-builds.org/docs/archives/ + # TODO: when CentOS 7 is phased out and tar>1.28 is everywhere, replace sort step + # in the pipe with tar-flag '--sort=name' and place LC_ALL in front of tar. + tar_cmd = [ + # print names of all files and folders excluding .git directory + 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', + # reset access and modification timestamps + '-exec', 'touch', '-t 197001010100', '{}', '\;', '|', + # sort file list + 'LC_ALL=C', 'sort', '--zero-terminated', '|', + # create tarball in GNU format with ownership reset + 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', + '--null', '--files-from', '-', '|', + # compress tarball with gzip without original file name and timestamp + 'gzip', '--no-name', '>', archive_path + ] run_shell_cmd(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True) # cleanup (repo_name dir does not exist in dry run mode) remove(tmpdir) - return targetpath + return archive_path def move_file(path, target_path, force_in_dry_run=False): From 46e61c85d881dc081936347930cfa44bdbd42bde Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 02:52:25 +0100 Subject: [PATCH 105/171] update FileToolsTest.test_github_get_source_tarball_from_git tests with reproducible tar commands --- test/framework/filetools.py | 63 +++++++++++++++---------------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3adb736586..8d823ef59c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2802,26 +2802,25 @@ def run_check(): 'git_repo': 'git@github.com:easybuilders/testrepository.git', 'test_prefix': self.test_prefix, } + reprod_tar_cmd_pattern = ( + r' running shell command "find {} -name \".git\" -prune -o -print0 -exec touch -t 197001010100 {{}} \; |' + r' LC_ALL=C sort --zero-terminated | tar --create --no-recursion --owner=0 --group=0 --numeric-owner' + r' --format=gnu --null --files-from - | gzip --no-name > %(test_prefix)s/target/test.tar.gz' + ) expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', r" \(in .*/tmp.*\)", - r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("test123"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2829,12 +2828,9 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2844,9 +2840,9 @@ def run_check(): ' running shell command "git clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", - ]) % git_repo + ]) % string_args run_check() git_config['extra_config_params'] = [ @@ -2858,21 +2854,18 @@ def run_check(): + ' clone --depth 1 --branch tag_for_tests --recursive' + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", - ]) % git_repo + ]) % string_args run_check() del git_config['recurse_submodules'] del git_config['extra_config_params'] git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar' - r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip' - r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion' - r' --files-from -"', + r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2881,28 +2874,22 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86 && git submodule update --init --recursive"', + r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86"', + r' running shell command "git checkout 8456f86"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2914,9 +2901,9 @@ def run_check(): r" \(in /.*\)", r' running shell command "git checkout 8456f86"', r" \(in /.*/testrepository\)", - r' running shell command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in /.*\)", - ]) % git_repo + ]) % string_args run_check() # Test with real data. From 8c76561bf026831bfb7ff065a923636f20dd4b1f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 29 Feb 2024 09:56:49 +0100 Subject: [PATCH 106/171] remove TODO about sort option in tar as it is not supported across implementations --- easybuild/tools/filetools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5b7406fffa..470e347e88 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2676,8 +2676,6 @@ def get_source_tarball_from_git(filename, target_dir, git_config): else: # create reproducible archive # see https://reproducible-builds.org/docs/archives/ - # TODO: when CentOS 7 is phased out and tar>1.28 is everywhere, replace sort step - # in the pipe with tar-flag '--sort=name' and place LC_ALL in front of tar. tar_cmd = [ # print names of all files and folders excluding .git directory 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', From 4fbf38fd314e975c112a78f7bee6dff5bb08dbc6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 29 Feb 2024 16:10:33 +0100 Subject: [PATCH 107/171] add support for --from-commit --- easybuild/framework/easyconfig/tools.py | 55 ++++++----- easybuild/tools/config.py | 3 +- easybuild/tools/github.py | 126 +++++++++++++++++++++--- easybuild/tools/options.py | 10 +- easybuild/tools/robot.py | 8 +- test/framework/easyconfig.py | 35 +++++++ test/framework/robot.py | 44 ++++++++- 7 files changed, 234 insertions(+), 47 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 73393040c6..711e14d29e 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -60,7 +60,8 @@ from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO -from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data +from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit +from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.github import fetch_files_from_pr from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict @@ -310,7 +311,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): return paths -def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=None): +def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, from_commit=None, review_pr=None): """Obtain alternative paths for easyconfig files.""" # paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be @@ -321,18 +322,20 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None, review_pr=Non tweaked_ecs_paths = (os.path.join(tmpdir, 'tweaked_easyconfigs'), os.path.join(tmpdir, 'tweaked_dep_easyconfigs')) - # paths where files touched in PRs will be downloaded to, - # which are picked up via 'pr_paths' build option in fetch_files_from_pr - pr_paths = [] + # paths where files touched in commit/PRs will be downloaded to, + # which are picked up via 'extra_ec_paths' build option in fetch_files_from_pr + extra_ec_paths = [] if from_prs: - pr_paths = from_prs[:] - if review_pr and review_pr not in pr_paths: - pr_paths.append(review_pr) + extra_ec_paths = from_prs[:] + if review_pr and review_pr not in extra_ec_paths: + extra_ec_paths.append(review_pr) + if extra_ec_paths: + extra_ec_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in extra_ec_paths] - if pr_paths: - pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in pr_paths] + if from_commit: + extra_ec_paths.append(os.path.join(tmpdir, 'files_commit_' + from_commit)) - return tweaked_ecs_paths, pr_paths + return tweaked_ecs_paths, extra_ec_paths def det_easyconfig_paths(orig_paths): @@ -346,27 +349,31 @@ def det_easyconfig_paths(orig_paths): except ValueError: raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.") + from_commit = build_option('from_commit') robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] + commit_files, pr_files = [], [] if from_prs: - pr_files = [] for pr in from_prs: - # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option, - # which corresponds to the list of PR paths returned by alt_easyconfig_paths + # path to where easyconfig files should be downloaded is determined + # via 'extra_ec_paths' build options, + # which corresponds to the list of commit/PR paths returned by alt_easyconfig_paths pr_files.extend(fetch_easyconfigs_from_pr(pr)) - - if ec_files: - # replace paths for specified easyconfigs that are touched in PR - for i, ec_file in enumerate(ec_files): - for pr_file in pr_files: - if ec_file == os.path.basename(pr_file): - ec_files[i] = pr_file - else: - # if no easyconfigs are specified, use all the ones touched in the PR - ec_files = [path for path in pr_files if path.endswith('.eb')] + elif from_commit: + commit_files = fetch_easyconfigs_from_commit(from_commit, files=ec_files) + + if ec_files: + # replace paths for specified easyconfigs that are touched in commit/PRs + for i, ec_file in enumerate(ec_files): + for file in commit_files + pr_files: + if ec_file == os.path.basename(file): + ec_files[i] = file + else: + # if no easyconfigs are specified, use all the ones touched in the commit/PRs + ec_files = [path for path in commit_files + pr_files if path.endswith('.eb')] filter_ecs = build_option('filter_ecs') if filter_ecs: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ee8eb5fc53..ebc4f4a7d1 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -220,6 +220,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'filter_env_vars', 'filter_rpath_sanity_libs', 'force_download', + 'from_commit', 'git_working_dirs_path', 'github_user', 'github_org', @@ -403,7 +404,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'build_specs', 'command_line', 'external_modules_metadata', - 'pr_paths', + 'extra_ec_paths', 'robot_path', 'valid_module_classes', 'valid_stops', diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 96ed4d6492..0cfc181120 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -54,7 +54,7 @@ from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_framework_files +from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files from easybuild.tools.filetools import det_patched_files, download_file, extract_file from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen @@ -355,7 +355,7 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token= return res -def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_MAIN, path=None, github_user=None): +def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, account=GITHUB_EB_MAIN, path=None, github_user=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download @@ -364,7 +364,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M :param path: path to extract to :param github_user: name of GitHub user to use """ - if branch is None: + if branch is None and commit is None: branch = pick_default_branch(account) # make sure path exists, create it if necessary @@ -375,9 +375,16 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M path = os.path.join(path, account) mkdir(path, parents=True) - extracted_dir_name = '%s-%s' % (repo, branch) - base_name = '%s.tar.gz' % branch - latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user) + if commit: + extracted_dir_name = '%s-%s' % (repo, commit) + base_name = '%s.tar.gz' % commit + latest_commit_sha = commit + elif branch: + extracted_dir_name = '%s-%s' % (repo, branch) + base_name = '%s.tar.gz' % branch + latest_commit_sha = fetch_latest_commit_sha(repo, account, branch, github_user=github_user) + else: + raise EasyBuildError("Either branch or commit should be specified in download_repo") expected_path = os.path.join(path, extracted_dir_name) latest_sha_path = os.path.join(expected_path, 'latest-sha') @@ -401,11 +408,22 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, account=GITHUB_EB_M # check if extracted_path exists if not os.path.isdir(extracted_path): - raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch) + error_msg = "%s should exist and contain the repo %s " % (extracted_path, repo) + if branch: + error_msg += "at branch " + branch + elif commit: + error_msg += "at commit " + commit + raise EasyBuildError(error_msg) write_file(latest_sha_path, latest_commit_sha, forced=True) - _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) + log_msg = "Repo %s at %%s extracted into %s" % (repo, extracted_path) + if branch: + log_msg = log_msg % ('branch ' + branch) + elif commit: + log_msg = log_msg % ('commit ' + commit) + _log.debug(log_msg) + return extracted_path @@ -450,14 +468,14 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if path is None: if github_repo == GITHUB_EASYCONFIGS_REPO: - pr_paths = build_option('pr_paths') - if pr_paths: + extra_ec_paths = build_option('extra_ec_paths') + if extra_ec_paths: # figure out directory for this specific PR (see also alt_easyconfig_paths) - cands = [p for p in pr_paths if p.endswith('files_pr%s' % pr)] + cands = [p for p in extra_ec_paths if p.endswith('files_pr%s' % pr)] if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, pr_paths) + raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, extra_ec_paths) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) @@ -567,6 +585,90 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) +def fetch_files_from_commit(commit, files=None, path=None, github_account=None, github_repo=None): + """ + Fetch files from a specific commit. + + If 'files' is None, all files touched in the commit are used. + """ + if github_account is None: + github_account = build_option('pr_target_account') + + if github_repo is None: + github_repo = GITHUB_EASYCONFIGS_REPO + + if path is None: + if github_repo == GITHUB_EASYCONFIGS_REPO: + extra_ec_paths = build_option('extra_ec_paths') + if extra_ec_paths: + # figure out directory for this specific commit (see also alt_easyconfig_paths) + cands = [p for p in extra_ec_paths if p.endswith('files_commit_' + commit)] + if len(cands) == 1: + path = cands[0] + else: + raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", commit, extra_ec_paths) + + elif github_repo == GITHUB_EASYBLOCKS_REPO: + path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit) + else: + raise EasyBuildError("Unknown repo: %s" % github_repo) + + # if no files are specified, determine which files are touched in commit + if not files: + diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff') + diff_fn = os.path.basename(diff_url) + diff_filepath = os.path.join(path, diff_fn) + download_file(diff_fn, diff_url, diff_filepath, forced=True) + diff_txt = read_file(diff_filepath) + _log.debug("Diff for commit %s:\n%s", commit, diff_txt) + + files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True) + _log.debug("List of patched files for commit %s: %s", commit, files) + + # download tarball for specific commit + repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account) + + if github_repo == GITHUB_EASYCONFIGS_REPO: + files_subdir = 'easybuild/easyconfigs/' + elif github_repo == GITHUB_EASYBLOCKS_REPO: + files_subdir = 'easybuild/easyblocks/' + + # copy specified files to directory where they're expected to be found + file_paths = [] + for file in files: + + # if only filename is specified, we need to determine the file path + if file == os.path.basename(file): + src_path = None + for (dirpath, dirnames, filenames) in os.walk(repo_commit, topdown=True): + if file in filenames: + src_path = os.path.join(dirpath, file) + break + else: + src_path = os.path.join(repo_commit, file) + + # strip of leading subdirectory like easybuild/easyconfigs/ or easybuild/easyblocks/ + # because that's what expected by robot_find_easyconfig + if file.startswith(files_subdir): + file = file[len(files_subdir):] + + # if file is found, copy it to dedicated directory; + # if not, just skip it (may be an easyconfig file in local directory); + if src_path and os.path.exists(src_path): + target_path = os.path.join(path, file) + copy_file(src_path, target_path) + file_paths.append(target_path) + else: + _log.info("File %s not found in %s, so ignoring it...", file, repo_commit) + + return file_paths + + +def fetch_easyconfigs_from_commit(commit, files=None, path=None): + """Fetch specified easyconfig files from a specific commit.""" + return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYCONFIGS_REPO) + + def create_gist(txt, fn, descr=None, github_user=None, github_token=None): """Create a gist with the provided text.""" diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 0ea04c7672..f99a5ac284 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -702,6 +702,7 @@ def github_options(self): 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), + 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'PR#'}), 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), @@ -1588,10 +1589,11 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs - tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs, - review_pr=review_pr) + tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(tmpdir, tweaked_ecs=tweaked_ecs, from_prs=from_prs, + from_commit=eb_go.options.from_commit, + review_pr=review_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query - robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_paths, auto_robot=auto_robot) + robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, extra_ec_paths, auto_robot=auto_robot) log.debug("Full robot path: %s", robot_path) if not robot_path: @@ -1605,7 +1607,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), - 'pr_paths': pr_paths, + 'extra_ec_paths': extra_ec_paths, 'robot_path': robot_path, 'silent': testing or new_update_opt, 'try_to_generate': try_to_generate, diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..e49f788d53 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -54,7 +54,7 @@ _log = fancylogger.getLogger('tools.robot', fname=False) -def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=False): +def det_robot_path(robot_paths_option, tweaked_ecs_paths, extra_ec_paths, auto_robot=False): """Determine robot path.""" robot_path = robot_paths_option[:] _log.info("Using robot path(s): %s", robot_path) @@ -70,9 +70,9 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, pr_paths, auto_robot=F _log.info("Prepended list of robot search paths with %s and appended with %s: %s", tweaked_ecs_path, tweaked_ecs_deps_path, robot_path) - if pr_paths is not None: - robot_path.extend(pr_paths) - _log.info("Extended list of robot search paths with %s: %s", pr_paths, robot_path) + if extra_ec_paths is not None: + robot_path.extend(extra_ec_paths) + _log.info("Extended list of robot search paths with %s: %s", extra_ec_paths, robot_path) return robot_path diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 905cf6c15b..2cadf46049 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -733,6 +733,41 @@ def test_tweaking(self): # cleanup os.remove(tweaked_fn) + def test_alt_easyconfig_paths(self): + """Test alt_easyconfig_paths function that collects list of additional paths for easyconfig files.""" + + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix) + self.assertEqual(tweaked_ecs_path, None) + self.assertEqual(extra_ecs_path, []) + + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + self.assertTrue(tweaked_ecs_path) + self.assertTrue(isinstance(tweaked_ecs_path, tuple)) + self.assertEqual(len(tweaked_ecs_path), 2) + self.assertEqual(tweaked_ecs_path[0], os.path.join(self.test_prefix, 'tweaked_easyconfigs')) + self.assertEqual(tweaked_ecs_path[1], os.path.join(self.test_prefix, 'tweaked_dep_easyconfigs')) + self.assertEqual(extra_ecs_path, []) + + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456]) + self.assertEqual(tweaked_ecs_path, None) + self.assertTrue(extra_ecs_path) + self.assertTrue(isinstance(extra_ecs_path, list)) + self.assertEqual(len(extra_ecs_path), 2) + self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123')) + self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456')) + + review_pr_path = os.path.join(self.test_prefix, 'review_pr') + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456], + review_pr=789, from_commit='c0ff33') + self.assertEqual(tweaked_ecs_path, None) + self.assertTrue(extra_ecs_path) + self.assertTrue(isinstance(extra_ecs_path, list)) + self.assertEqual(len(extra_ecs_path), 4) + self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123')) + self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456')) + self.assertEqual(extra_ecs_path[2], os.path.join(self.test_prefix, 'files_pr789')) + self.assertEqual(extra_ecs_path[3], os.path.join(self.test_prefix, 'files_commit_c0ff33')) + def test_tweak_multiple_tcs(self): """Test that tweaking variables of ECs from multiple toolchains works""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') diff --git a/test/framework/robot.py b/test/framework/robot.py index 4f226bae35..4a23a21500 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -713,6 +713,46 @@ def test_search_paths(self): regex = re.compile(r"^ \* %s$" % os.path.join(self.test_prefix, test_ec), re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + def test_github_det_easyconfig_paths_from_commit(self): + """Test det_easyconfig_paths function in combination with --from-commit.""" + + test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + commit = '589282cf52609067616fc2a522f8e4b81f809cb7' + args = [ + os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), # absolute path + 'toy-0.0-iter.eb', # relative path, available via robot search path + # commit in which ReFrame-4.3.2.eb was added, see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits + '--from-commit', commit, + 'ReFrame-4.3.2.eb', # easyconfig included in commit, should be resolved via robot search path + '--dry-run', + '--robot', + '--robot=%s' % test_ecs_path, + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + '--tmpdir=%s' % self.test_prefix, + ] + + self.mock_stderr(True) + outtxt = self.eb_main(args, raise_error=True) + stderr = self.get_stderr() + self.mock_stderr(False) + + self.assertFalse(stderr) + + # full path doesn't matter (helps to avoid failing tests due to resolved symlinks) + test_ecs_path = os.path.join('.*', 'test', 'framework', 'easyconfigs', 'test_ecs') + + modules = [ + (test_ecs_path, 'toy/0.0'), + (test_ecs_path, 'toy/0.0-iter'), + (os.path.join(self.test_prefix, '.*', 'files_commit_%s' % commit), 'ReFrame/4.3.2'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path_prefix, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + def test_github_det_easyconfig_paths_from_pr(self): """Test det_easyconfig_paths function, with --from-pr enabled as well.""" if self.github_token is None: @@ -1082,8 +1122,8 @@ def test_tweak_robotpath(self): test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') # Create directories to store the tweaked easyconfigs - tweaked_ecs_paths, pr_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) - robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_paths, auto_robot=True) + tweaked_ecs_paths, extra_ec_paths = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, extra_ec_paths, auto_robot=True) init_config(build_options={ 'valid_module_classes': module_classes(), From 4363f9f371cbb94623e0be62c5ca3de37469fd88 Mon Sep 17 00:00:00 2001 From: Rovanion Luckey Date: Fri, 1 Mar 2024 10:40:09 +0100 Subject: [PATCH 108/171] filetools: Fix missing raw string for \; in get_source_tarball_from_git. --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 470e347e88..330c89366b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2680,7 +2680,7 @@ def get_source_tarball_from_git(filename, target_dir, git_config): # print names of all files and folders excluding .git directory 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', # reset access and modification timestamps - '-exec', 'touch', '-t 197001010100', '{}', '\;', '|', + '-exec', 'touch', '-t 197001010100', '{}', r'\;', '|', # sort file list 'LC_ALL=C', 'sort', '--zero-terminated', '|', # create tarball in GNU format with ownership reset From 07d06af44eeb3cbc70879a25b6dc28f4433cf195 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 1 Mar 2024 17:23:14 +0100 Subject: [PATCH 109/171] fix test_github_merge_pr by using more recent easyconfigs PR --- test/framework/options.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 26daa0cc7a..a491936d78 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4806,26 +4806,29 @@ def test_github_merge_pr(self): self.assertEqual(stderr.strip(), expected_stderr) self.assertTrue(stdout.strip().endswith(expected_stdout), "'%s' ends with '%s'" % (stdout, expected_stdout)) - # full eligible merged PR, default target branch + # full eligible merged PR, default target branch; + # note: we frequently need to change to a more recent PR here, + # to avoid that this test starts failing because commit status is set to None for old commits del args[-1] - args[1] = '17065' + # easyconfig PR for EasyBuild v4.8.2 + args[1] = '19105' stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) expected_stdout = '\n'.join([ - "Checking eligibility of easybuilders/easybuild-easyconfigs PR #17065 for merging...", + "Checking eligibility of easybuilders/easybuild-easyconfigs PR #19105 for merging...", "* targets develop branch: OK", "* test suite passes: OK", "* last test report is successful: OK", "* no pending change requests: OK", "* approved review: OK (by SebastianAchilles)", - "* milestone is set: OK (4.7.1)", + "* milestone is set: OK (4.9.0)", "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", '', - "[DRY RUN] Adding comment to easybuild-easyconfigs issue #17065: 'Going in, thanks @boegel!'", - "[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #17065", + "[DRY RUN] Adding comment to easybuild-easyconfigs issue #19105: 'Going in, thanks @boegel!'", + "[DRY RUN] Merged easybuilders/easybuild-easyconfigs pull request #19105", ]) expected_stderr = '' self.assertEqual(stderr.strip(), expected_stderr) From ff4ff4934bd9a21be971d71a95685c67394ae72f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 1 Mar 2024 16:58:14 +0100 Subject: [PATCH 110/171] update easyblocks for toy extensions to make sure that asynchronous installation command is run in correct working directory --- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 15fe5773aa..28792ff4df 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -93,7 +93,7 @@ def run_async(self, thread_pool): cmd = f"echo 'no sources for {self.name}'" return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), - fail_on_error=False, task_id=task_id) + fail_on_error=False, task_id=task_id, work_dir=os.getcwd()) def postrun(self): """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index fb842bfe06..19b19d32dd 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -170,7 +170,7 @@ def run_async(self, thread_pool): cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) task_id = f'ext_{self.name}_{self.version}' return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), - fail_on_error=False, task_id=task_id) + fail_on_error=False, task_id=task_id, work_dir=os.getcwd()) def postrun(self): """ From cb6905601739d6f3437a29777a861eefc90ac96c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 1 Mar 2024 17:43:32 +0100 Subject: [PATCH 111/171] add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration --- .github/workflows/container_tests_apptainer.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 35c26c26c9..137b297379 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -29,10 +29,18 @@ jobs: - name: install OS & Python packages run: | # for building CentOS 7 container images - sudo apt-get install rpm - sudo apt-get install dnf + APT_PKGS="rpm dnf" # for modules tool - sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then From 2266b7166ccffd5826396955ca2b485657cca663 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 1 Mar 2024 17:04:16 +0000 Subject: [PATCH 112/171] append to `APT_PKGS` instead of overwriting it --- .github/workflows/container_tests_apptainer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 137b297379..b7fc2a8418 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -31,7 +31,7 @@ jobs: # for building CentOS 7 container images APT_PKGS="rpm dnf" # for modules tool - APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + APT_PKGS+="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" # Avoid apt-get update, as we don't really need it, # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) From 05ab883213870c7cf86d97d2be2fa05792063aba Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 1 Mar 2024 17:06:45 +0000 Subject: [PATCH 113/171] missing space --- .github/workflows/container_tests_apptainer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index b7fc2a8418..77d2a4a395 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -31,7 +31,7 @@ jobs: # for building CentOS 7 container images APT_PKGS="rpm dnf" # for modules tool - APT_PKGS+="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" # Avoid apt-get update, as we don't really need it, # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) From e3308507256a045b691c5d674a221804502f6d9a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 2 Mar 2024 10:44:33 +0100 Subject: [PATCH 114/171] add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration --- .github/workflows/container_tests_apptainer.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 1f46f2f5f6..1539221489 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -29,10 +29,18 @@ jobs: - name: install OS & Python packages run: | # for building CentOS 7 container images - sudo apt-get install rpm - sudo apt-get install dnf + APT_PKGS="rpm dnf" # for modules tool - sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev + APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" + + # Avoid apt-get update, as we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + if ! sudo apt-get install $APT_PKGS; then + # Try to update cache, then try again to resolve 404s of old packages + sudo apt-get update -yqq || true + sudo apt-get install $APT_PKGS + fi + # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then From 5377e40175229206de64bf71904b688c65dbcbfe Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 2 Mar 2024 13:55:15 +0000 Subject: [PATCH 115/171] replace `log.warn` with `log.warning` --- easybuild/tools/filetools.py | 2 +- test/framework/build_log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1208be1d07..d0959723f8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2240,7 +2240,7 @@ def det_size(path): if os.path.exists(fullpath): installsize += os.path.getsize(fullpath) except OSError as err: - _log.warn("Could not determine install size: %s" % err) + _log.warning("Could not determine install size: %s" % err) return installsize diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 21755b62b3..dcaa1620c3 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -208,7 +208,7 @@ def test_log_levels(self): log.error('kaput') log.deprecated('almost kaput', '10000000000000') log.raiseError = True - log.warn('this is a warning') + log.warning('this is a warning') log.info('fyi') log.debug('gdb') log.devel('tmi') From 57839104045f839476536330a0cb53da8d586bc4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Mar 2024 12:03:03 +0100 Subject: [PATCH 116/171] add support to run_shell_cmd for providing list of answers in qa_patterns + fix output for hooks triggered for interactive shell commands run with run_shell_cmd --- easybuild/tools/run.py | 20 ++++++++++++++++++-- test/framework/run.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c9a4b3d06f..ba40b704ac 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -319,7 +319,11 @@ def to_cmd_str(cmd): if with_hooks: hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir}) + kwargs = { + 'interactive': bool(qa_patterns), + 'work_dir': work_dir, + } + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs) if hook_res: cmd, old_cmd = hook_res, cmd cmd_str = to_cmd_str(cmd) @@ -375,10 +379,21 @@ def to_cmd_str(cmd): # only consider answering questions if there's new output beyond additional whitespace if qa_patterns: - for question, answer in qa_patterns: + for question, answers in qa_patterns: + question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): + # if answer is specified as a list, we take the first item as current answer, + # and add it to the back of the list (so we cycle through answers) + if isinstance(answers, list): + answer = answers.pop(0) + answers.append(answer) + elif isinstance(answers, str): + answer = answers + else: + raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 @@ -437,6 +452,7 @@ def to_cmd_str(cmd): if with_hooks: run_hook_kwargs = { 'exit_code': res.exit_code, + 'interactive': bool(qa_patterns), 'output': res.output, 'stderr': res.stderr, 'work_dir': res.work_dir, diff --git a/test/framework/run.py b/test/framework/run.py index aa22ef88fd..6fb8d2b072 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -996,6 +996,27 @@ def test_run_cmd_qa_answers(self): self.assertEqual(out, "question\nanswer1\nquestion\nanswer2\n" * 2) self.assertEqual(ec, 0) + def test_run_shell_cmd_qa_answers(self): + """Test providing list of answers for a question in run_shell_cmd.""" + + cmd = "echo question; read x; echo $x; " * 2 + qa = [("question", ["answer1", "answer2"])] + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n") + self.assertEqual(res.exit_code, 0) + + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd, qa_patterns=[('question', 1)]) + + # test cycling of answers + cmd = cmd * 2 + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.output, "question\nanswer1\nquestion\nanswer2\n" * 2) + self.assertEqual(res.exit_code, 0) + def test_run_cmd_simple(self): """Test return value for run_cmd in 'simple' mode.""" with self.mocked_stdout_stderr(): @@ -1124,6 +1145,14 @@ def test_run_cmd_dry_run(self): self.assertEqual(read_file(outfile), "This is always echoed\n") # Q&A commands + self.mock_stdout(True) + run_shell_cmd("some_qa_cmd", qa_patterns=[('question1', 'answer1')]) + stdout = self.get_stdout() + self.mock_stdout(False) + + expected = """ running interactive shell command "some_qa_cmd"\n""" + self.assertIn(expected, stdout) + self.mock_stdout(True) run_cmd_qa("some_qa_cmd", {'question1': 'answer1'}) stdout = self.get_stdout() @@ -1565,6 +1594,17 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ]) self.assertEqual(stdout, expected_stdout) + with self.mocked_stdout_stderr(): + run_shell_cmd("sleep 2; make", qa_patterns=[('q', 'a')]) + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook interactive 'sleep 2; make' in %s" % cwd, + "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + with self.mocked_stdout_stderr(): run_cmd_qa("sleep 2; make", qa={}) stdout = self.get_stdout() From e9cec346425783205431ebe296d186d4df3aad6e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 14:58:16 -0500 Subject: [PATCH 117/171] Eliminate use of distutils.version.StrictVersion distutils was removed in Python 3.12. The only reason EasyBuild uses StrictVersion is that it orders beta/rc versions before the released version, unlike LooseVersion. E.g. 5.0.0-beta < 5.0.0 (but > for LooseVersion). So a new method `is_earlier_or_prerelease(self, other, markers)` was added to LooseVersion to handle that particular case. Addresses part of #3963 --- easybuild/tools/__init__.py | 9 --------- easybuild/tools/loose_version.py | 16 ++++++++++++++++ easybuild/tools/modules.py | 27 ++++++++++----------------- test/framework/modules.py | 14 ++++++++------ test/framework/modulestool.py | 6 +++--- test/framework/utilities_test.py | 4 ++++ 6 files changed, 41 insertions(+), 35 deletions(-) diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index dee4fc0d12..bd36b15edb 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -37,14 +37,5 @@ __path__ = __import__('pkgutil').extend_path(__path__, __name__) -import distutils.version import warnings from easybuild.tools.loose_version import LooseVersion # noqa(F401) - - -class StrictVersion(distutils.version.StrictVersion): - """Temporary wrapper over distuitls StrictVersion that silences the deprecation warning""" - def __init__(self, *args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - distutils.version.StrictVersion.__init__(self, *args, **kwargs) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index 1855fee74a..db374b20ca 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -53,6 +53,22 @@ def version(self): """Readonly access to the parsed version (list or None)""" return self._version + def is_earlier_or_prerelease(self, other, markers): + """Check if this is an earlier version or prerelease of other + + Markers is a list of strings that denote a prerelease + """ + if isinstance(other, str): + vstring = other + else: + vstring = other._vstring + if self._vstring.startswith(vstring): + prerelease = self._vstring[len(vstring):] + for marker in markers: + if prerelease.startswith(marker): + return True + return self < other + def __str__(self): return self._vstring diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 75a1022b50..c2bbf71065 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -42,7 +42,7 @@ import shlex from easybuild.base import fancylogger -from easybuild.tools import StrictVersion +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS @@ -144,11 +144,11 @@ class ModulesTool(object): COMMAND_SHELL = None # option to determine the version VERSION_OPTION = '--version' - # minimal required version (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion)) + # minimal required version (cannot include -beta or rc) REQ_VERSION = None # deprecated version limit (support for versions below this version is deprecated) DEPR_VERSION = None - # maximum version allowed (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion)) + # maximum version allowed (cannot include -beta or rc) MAX_VERSION = None # the regexp, should have a "version" group (multiline search) VERSION_REGEXP = None @@ -239,14 +239,6 @@ def set_and_check_version(self): if res: self.version = res.group('version') self.log.info("Found %s version %s", self.NAME, self.version) - - # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid), - # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release - self.version = self.version.replace('rc', 'b').replace('-beta', 'b1') - if len(self.version.split('.')) > 3: - self.version = '.'.join(self.version.split('.')[:3]) - - self.log.info("Converted actual version to '%s'" % self.version) else: raise EasyBuildError("Failed to determine %s version from option '%s' output: %s", self.NAME, self.VERSION_OPTION, txt) @@ -259,9 +251,10 @@ def set_and_check_version(self): elif build_option('modules_tool_version_check'): self.log.debug("Checking whether %s version %s meets requirements", self.NAME, self.version) + version = LooseVersion(self.version) if self.REQ_VERSION is not None: self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION) - if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION): + if version.is_earlier_or_prerelease(self.REQ_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s", self.NAME, self.REQ_VERSION, self.version) else: @@ -269,14 +262,14 @@ def set_and_check_version(self): if self.DEPR_VERSION is not None: self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION) - if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION): + if version.is_earlier_or_prerelease(self.DEPR_VERSION, ['rc', '-beta']): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) - if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION): + if version.is_earlier_or_prerelease(self.MAX_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s", self.NAME, self.MAX_VERSION, self.version) else: @@ -1390,7 +1383,7 @@ def available(self, mod_name=None, extra_args=None): if extra_args is None: extra_args = [] # make hidden modules visible (requires Environment Modules 4.6.0) - if StrictVersion(self.version) >= StrictVersion('4.6.0'): + if LooseVersion(self.version) >= LooseVersion('4.6.0'): extra_args.append(self.SHOW_HIDDEN_OPTION) return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args) @@ -1440,11 +1433,11 @@ def __init__(self, *args, **kwargs): setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False) super(Lmod, self).__init__(*args, **kwargs) - version = StrictVersion(self.version) + version = LooseVersion(self.version) self.supports_depends_on = True # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html - if version >= '8.7.12': + if version >= LooseVersion('8.7.12'): self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') else: self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') diff --git a/test/framework/modules.py b/test/framework/modules.py index a849148bdf..3e9c7343ba 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -42,7 +42,7 @@ import easybuild.tools.modules as mod from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig.easyconfig import EasyConfig -from easybuild.tools import StrictVersion +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir @@ -226,15 +226,16 @@ def test_avail(self): # all test modules are accounted for ms = self.modtool.available() + version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('5.7.5'): + if isinstance(self.modtool, Lmod) and not version.is_prerelease_or_earlier('5.7.5', ['rc']): # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + and not version.is_prerelease_or_earlier('4.6.0', ['-beta'])): # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) @@ -314,7 +315,8 @@ def test_exist(self): avail_mods = self.modtool.available() self.assertIn('Java/1.8.0_181', avail_mods) - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): + version = LooseVersion(self.modtool.version) + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): self.assertIn('Java/1.8', avail_mods) self.assertIn('Java/site_default', avail_mods) self.assertIn('JavaAlias', avail_mods) @@ -374,7 +376,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True]) # also check with .modulerc.lua for Lmod 7.8 or newer - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.8'): + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.8', ['rc']): shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir) reset_module_caches() @@ -406,7 +408,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) # Test alias in home directory .modulerc - if isinstance(self.modtool, Lmod) and StrictVersion(self.modtool.version) >= StrictVersion('7.0'): + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): # Required or temporary HOME would be in MODULEPATH already self.init_testmods() # Sanity check: Module aliases don't exist yet diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index f43a91e3b3..3c3eae74f5 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -36,7 +36,7 @@ from unittest import TextTestRunner from easybuild.base import fancylogger -from easybuild.tools import modules, StrictVersion +from easybuild.tools import modules, LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.modules import EnvironmentModules, Lmod @@ -76,7 +76,7 @@ def test_mock(self): mmt = MockModulesTool(mod_paths=[], testing=True) # the version of the MMT is the commandline option - self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) + self.assertEqual(mmt.version, LooseVersion(MockModulesTool.VERSION_OPTION)) cmd_abspath = which(MockModulesTool.COMMAND) @@ -100,7 +100,7 @@ def test_environment_command(self): bmmt = BrokenMockModulesTool(mod_paths=[], testing=True) cmd_abspath = which(MockModulesTool.COMMAND) - self.assertEqual(bmmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) + self.assertEqual(bmmt.version, LooseVersion(MockModulesTool.VERSION_OPTION)) self.assertEqual(bmmt.cmd, cmd_abspath) # clean it up diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index ba4766f302..6e9421681f 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -143,6 +143,10 @@ def test_LooseVersion(self): # Careful here: 1.0 > 1 !!! self.assertGreater(LooseVersion('1.0'), LooseVersion('1')) self.assertLess(LooseVersion('1'), LooseVersion('1.0')) + # checking prereleases + self.assertGreater(LooseVersion('4.0.0-beta'), LooseVersion('4.0.0')) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['-beta'], True) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['rc'], False) # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 From 7d9762088be7c1db05166aff1336e6b0ecb22346 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:07:02 -0500 Subject: [PATCH 118/171] Remove unused warnings imported, spotted by flake8. --- easybuild/tools/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index bd36b15edb..cc7eadb99f 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -37,5 +37,4 @@ __path__ = __import__('pkgutil').extend_path(__path__, __name__) -import warnings from easybuild.tools.loose_version import LooseVersion # noqa(F401) From 5c6b7f7f1a60f548b4fd7a69a6360695c93e21cb Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:27:09 -0500 Subject: [PATCH 119/171] Eliminate `distutils.util.strtobool`. It's only used once so I open-coded it. A stricter but not backwards compatible check would be to only allow 'True' and 'False'. --- easybuild/framework/easyconfig/types.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index fa9829e312..2cf9b77058 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -31,7 +31,6 @@ * Caroline De Brouwer (Ghent University) * Kenneth Hoste (Ghent University) """ -from distutils.util import strtobool from easybuild.base import fancylogger from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS @@ -280,7 +279,14 @@ def to_toolchain_dict(spec): res = {'name': spec[0].strip(), 'version': spec[1].strip()} # 3-element list elif len(spec) == 3: - res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': strtobool(spec[2].strip())} + hidden = spec[2].strip().lower() + if hidden in {'yes', 'true', 't', 'y', '1'}: + hidden = True + elif hidden in {'no', 'false', 'f', 'n', '0'}: + hidden = False + else: + raise EasyBuildError("Invalid truth value %s", hidden) + res = {'name': spec[0].strip(), 'version': spec[1].strip(), 'hidden': hidden} else: raise EasyBuildError("Can not convert list %s to toolchain dict. Expected 2 or 3 elements", spec) From 03c0b3d238dfa1e804abc58d2270e79311cfafd6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:54:47 -0500 Subject: [PATCH 120/171] Fix spelling of method call. --- test/framework/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 3e9c7343ba..c51312698a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -228,14 +228,14 @@ def test_avail(self): ms = self.modtool.available() version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_prerelease_or_earlier('5.7.5', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('5.7.5', ['rc']): # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and not version.is_prerelease_or_earlier('4.6.0', ['-beta'])): + and not version.is_earlier_or_prerelease('4.6.0', ['-beta'])): # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) From 303f453b1b8ddb688e34ff6fede3cb9e57dcb64b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 15:56:33 -0500 Subject: [PATCH 121/171] Fix brackets for is_earlier_or_prerelease call --- test/framework/utilities_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 6e9421681f..6e7487bae8 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -145,8 +145,8 @@ def test_LooseVersion(self): self.assertLess(LooseVersion('1'), LooseVersion('1.0')) # checking prereleases self.assertGreater(LooseVersion('4.0.0-beta'), LooseVersion('4.0.0')) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['-beta'], True) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0'), ['rc'], False) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['-beta']), True) + self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['rc']), False) # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 From 9cd91f0ff5fd9321545c987484985d93ea08bfba Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 16:00:00 -0500 Subject: [PATCH 122/171] Check for EasyBuildError for strtobool replacement --- test/framework/type_checking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index 1eb2f19fd3..e52c3da603 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -317,9 +317,9 @@ def test_to_toolchain_dict(self): self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', 'False', '7']) # invalid truth value - errstr = "invalid truth value .*" - self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, "intel, 2015, foo") - self.assertErrorRegex(ValueError, errstr, to_toolchain_dict, ['gcc', '4', '7']) + errstr = "Invalid truth value .*" + self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, "intel, 2015, foo") + self.assertErrorRegex(EasyBuildError, errstr, to_toolchain_dict, ['gcc', '4', '7']) # missing keys self.assertErrorRegex(EasyBuildError, "Incorrect set of keys", to_toolchain_dict, {'name': 'intel'}) From 8f2265131883e0eb6b75c25b640228e0e097314b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Mar 2024 17:02:40 -0500 Subject: [PATCH 123/171] Use is_prerelase instead of is_earlier_or_prerelease. In the end this is clearer because we also need to test against a maximum version (but it is allowed to be a prerelease of that maximum version, as per StrictVersion semantics!) --- easybuild/tools/loose_version.py | 6 +++--- easybuild/tools/modules.py | 6 +++--- test/framework/modules.py | 10 +++++----- test/framework/utilities_test.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index db374b20ca..fb2e0fcb78 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -53,8 +53,8 @@ def version(self): """Readonly access to the parsed version (list or None)""" return self._version - def is_earlier_or_prerelease(self, other, markers): - """Check if this is an earlier version or prerelease of other + def is_prerelease(self, other, markers): + """Check if this is a prerelease of other Markers is a list of strings that denote a prerelease """ @@ -67,7 +67,7 @@ def is_earlier_or_prerelease(self, other, markers): for marker in markers: if prerelease.startswith(marker): return True - return self < other + return False def __str__(self): return self._vstring diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index c2bbf71065..ae358e3bbe 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -254,7 +254,7 @@ def set_and_check_version(self): version = LooseVersion(self.version) if self.REQ_VERSION is not None: self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION) - if version.is_earlier_or_prerelease(self.REQ_VERSION, ['rc', '-beta']): + if version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']) or version < self.REQ_VERSION: raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s", self.NAME, self.REQ_VERSION, self.version) else: @@ -262,14 +262,14 @@ def set_and_check_version(self): if self.DEPR_VERSION is not None: self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION) - if version.is_earlier_or_prerelease(self.DEPR_VERSION, ['rc', '-beta']): + if version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']) or version < self.DEPR_VERSION: depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) - if version.is_earlier_or_prerelease(self.MAX_VERSION, ['rc', '-beta']): + if not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']) and self.version > self.MAX_VERSION: raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s", self.NAME, self.MAX_VERSION, self.version) else: diff --git a/test/framework/modules.py b/test/framework/modules.py index c51312698a..efcad01d5a 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -228,14 +228,14 @@ def test_avail(self): ms = self.modtool.available() version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('5.7.5', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('5.7.5', ['rc']) and version >= '5.7.5': # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and not version.is_earlier_or_prerelease('4.6.0', ['-beta'])): + and not version.is_prerelease('4.6.0', ['-beta'])) and version >= '4.6.0': # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) @@ -316,7 +316,7 @@ def test_exist(self): avail_mods = self.modtool.available() self.assertIn('Java/1.8.0_181', avail_mods) version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) and version >= '7.0': self.assertIn('Java/1.8', avail_mods) self.assertIn('Java/site_default', avail_mods) self.assertIn('JavaAlias', avail_mods) @@ -376,7 +376,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True]) # also check with .modulerc.lua for Lmod 7.8 or newer - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.8', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.8', ['rc']) and version >= '7.8': shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir) reset_module_caches() @@ -408,7 +408,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) # Test alias in home directory .modulerc - if isinstance(self.modtool, Lmod) and not version.is_earlier_or_prerelease('7.0', ['rc']): + if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) or version >= '7.0': # Required or temporary HOME would be in MODULEPATH already self.init_testmods() # Sanity check: Module aliases don't exist yet diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 6e7487bae8..2e862e6ad6 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -145,8 +145,8 @@ def test_LooseVersion(self): self.assertLess(LooseVersion('1'), LooseVersion('1.0')) # checking prereleases self.assertGreater(LooseVersion('4.0.0-beta'), LooseVersion('4.0.0')) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['-beta']), True) - self.assertEqual(LooseVersion('4.0.0-beta').is_earlier_or_prerelease('4.0.0', ['rc']), False) + self.assertEqual(LooseVersion('4.0.0-beta').is_prerelease('4.0.0', ['-beta']), True) + self.assertEqual(LooseVersion('4.0.0-beta').is_prerelease('4.0.0', ['rc']), False) # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 From bce95ecea3f5b5bcbc244dfb2c39a3227bd5e2d8 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 7 Mar 2024 09:56:02 -0500 Subject: [PATCH 124/171] Apply suggestions from code review Allow `on` and `off` as well for compatibility with `distutils.strtobool` Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/framework/easyconfig/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 2cf9b77058..451d22af03 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -280,9 +280,9 @@ def to_toolchain_dict(spec): # 3-element list elif len(spec) == 3: hidden = spec[2].strip().lower() - if hidden in {'yes', 'true', 't', 'y', '1'}: + if hidden in {'yes', 'true', 't', 'y', '1', 'on'}: hidden = True - elif hidden in {'no', 'false', 'f', 'n', '0'}: + elif hidden in {'no', 'false', 'f', 'n', '0', 'off'}: hidden = False else: raise EasyBuildError("Invalid truth value %s", hidden) From afc36be43c0e1f8fe2bd8a44f948f4b44eab4472 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 19:48:32 +0000 Subject: [PATCH 125/171] ensure parameters in DEPRECATED_PARAMETERS are not treated as unknown --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6e0d2aa141..7f123be9dc 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -179,7 +179,7 @@ def triage_easyconfig_params(variables, ec): for key in variables: # validations are skipped, just set in the config - if key in ec: + if key in ec or key in DEPRECATED_PARAMETERS.keys(): ec_params[key] = variables[key] _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key])) elif key in REPLACED_PARAMETERS: @@ -658,7 +658,7 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if key in self._config.keys(): + if key in self._config.keys() or key in DEPRECATED_PARAMETERS.keys(): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) From b8290964c848d8868e792bfe33d3fc45242d09ae Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 20:07:29 +0000 Subject: [PATCH 126/171] update deprecated functionality docs url --- test/framework/build_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/build_log.py b/test/framework/build_log.py index dcaa1620c3..581a995c24 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -116,7 +116,7 @@ def test_easybuildlog(self): stderr = self.get_stderr() self.mock_stderr(False) - more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information" + more_info = "see https://docs.easybuild.io/deprecated-functionality/ for more information" expected_stderr = '\n\n'.join([ "\nWARNING: Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info, "\nWARNING: Deprecated functionality, will no longer work in v2.0: onemorewarning", From d2b3c7d17c0533814056bcb458d12666c1a13e10 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 20:09:25 +0000 Subject: [PATCH 127/171] remove period at end of easyconfig parameter deprecation warning --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 6e0d2aa141..f03f2549e7 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -121,7 +121,7 @@ def new_ec_method(self, key, *args, **kwargs): if key in DEPRECATED_PARAMETERS: depr_key = key key, ver = DEPRECATED_PARAMETERS[depr_key] - _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead." % (depr_key, key), ver) + _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver) if key in REPLACED_PARAMETERS: _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') return ec_method(self, key, *args, **kwargs) From 4b873ecff163612814ef032ab753e1cd14ca7f99 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Thu, 7 Mar 2024 20:15:34 +0000 Subject: [PATCH 128/171] update easybuild.readthedocs.io urls to use docs.easybuild.io --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- easybuild/tools/build_log.py | 2 +- easybuild/tools/github.py | 2 +- easybuild/tools/modules.py | 2 +- easybuild/tools/toolchain/compiler.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f03f2549e7..b7763d02d6 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -792,7 +792,7 @@ def local_var_naming(self, local_var_naming_check): msg = "Use of %d unknown easyconfig parameters detected %s: %s\n" % (cnt, in_fn, unknown_keys_msg) msg += "If these are just local variables please rename them to start with '%s', " % LOCAL_VAR_PREFIX msg += "or try using --fix-deprecated-easyconfigs to do this automatically.\nFor more information, see " - msg += "https://easybuild.readthedocs.io/en/latest/Easyconfig-files-local-variables.html ." + msg += "https://docs.easybuild.io/easyconfig-files-local-variables/ ." # always log a warning if local variable that don't follow recommended naming scheme are found self.log.warning(msg) @@ -830,7 +830,7 @@ def check_deprecated(self, path): depr_maj_ver = int(str(VERSION).split('.')[0]) + 1 depr_ver = '%s.0' % depr_maj_ver - more_info_depr_ec = " (see also http://easybuild.readthedocs.org/en/latest/Deprecated-easyconfigs.html)" + more_info_depr_ec = " (https://docs.easybuild.io/deprecated-easyconfigs)" self.log.deprecated(depr_msg, depr_ver, more_info=more_info_depr_ec, silent=build_option('silent')) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index f8529fff57..04bcaae16c 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -55,7 +55,7 @@ # allow some experimental experimental code EXPERIMENTAL = False -DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html' +DEPRECATED_DOC_URL = 'https://docs.easybuild.io/deprecated-functionality/' DRY_RUN_BUILD_DIR = None DRY_RUN_SOFTWARE_INSTALL_DIR = None diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 875724acb4..f366985b7f 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2178,7 +2178,7 @@ def check_github(): msg = '\n'.join([ '', "One or more checks FAILed, GitHub configuration not fully complete!", - "See http://easybuild.readthedocs.org/en/latest/Integration_with_GitHub.html#configuration for help.", + "See https://docs.easybuild.io/integration-with-github/#github_configuration for help.", '', ]) print_msg(msg, log=_log, prefix=False) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 75a1022b50..8c51ec57b5 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -945,7 +945,7 @@ def check_loaded_modules(self): "use the --allow-loaded-modules configuration option.", "To specify action to take when loaded modules are detected, use %s." % opt, '', - "See http://easybuild.readthedocs.io/en/latest/Detecting_loaded_modules.html for more information.", + "See https://docs.easybuild.io/detecting-loaded-modules/ for more information.", ]) action = build_option('detect_loaded_modules') diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 6387273571..b70ebed589 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -358,9 +358,9 @@ def _set_optimal_architecture(self, default_optarch=None): optarch_flags_str = "%soptarch flags" % ('', 'generic ')[use_generic] error_msg = "Don't know how to set %s for %s/%s! " % (optarch_flags_str, self.arch, self.cpu_family) error_msg += "Use --optarch='' to override (see " - error_msg += "http://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html " + error_msg += "https://docs.easybuild.io/controlling-compiler-optimization-flags/ " error_msg += "for details) and consider contributing your settings back (see " - error_msg += "http://easybuild.readthedocs.io/en/latest/Contributing.html)." + error_msg += "https://docs.easybuild.io/contributing/)." raise EasyBuildError(error_msg) def comp_family(self, prefix=None): From ec7be23acd1c7c6e72e0d2681024c5ca7a11adda Mon Sep 17 00:00:00 2001 From: Jasper Grimm <65227842+jfgrimm@users.noreply.github.com> Date: Thu, 7 Mar 2024 20:46:16 +0000 Subject: [PATCH 129/171] reinstate "see also" accidentally removed --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b7763d02d6..aab3674cd3 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -830,7 +830,7 @@ def check_deprecated(self, path): depr_maj_ver = int(str(VERSION).split('.')[0]) + 1 depr_ver = '%s.0' % depr_maj_ver - more_info_depr_ec = " (https://docs.easybuild.io/deprecated-easyconfigs)" + more_info_depr_ec = " (see also https://docs.easybuild.io/deprecated-easyconfigs)" self.log.deprecated(depr_msg, depr_ver, more_info=more_info_depr_ec, silent=build_option('silent')) From 63acc8f2112ed24d1b564067c82599c5d1c1c726 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 8 Mar 2024 10:31:22 +0100 Subject: [PATCH 130/171] Improve logging & handling of --optarch --- easybuild/tools/toolchain/compiler.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 32634032fc..5d9f025172 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -323,7 +323,7 @@ def _set_optimal_architecture(self, default_optarch=None): optarch = build_option('optarch') # --optarch is specified with flags to use - if optarch is not None and isinstance(optarch, dict): + if isinstance(optarch, dict): # optarch has been validated as complex string with multiple compilers and converted to a dictionary # first try module names, then the family in optarch current_compiler_names = (getattr(self, 'COMPILER_MODULE_NAME', []) + @@ -338,14 +338,12 @@ def _set_optimal_architecture(self, default_optarch=None): self.log.info("_set_optimal_architecture: no optarch found for compiler %s. Ignoring option.", current_compiler) - use_generic = False - if optarch is not None: - # optarch has been parsed as a simple string - if isinstance(optarch, string_type): - if optarch == OPTARCH_GENERIC: - use_generic = True - else: - raise EasyBuildError("optarch is neither an string or a dict %s. This should never happen", optarch) + if isinstance(optarch, string_type): + use_generic = (optarch == OPTARCH_GENERIC) + elif optarch is None: + use_generic = False + else: + raise EasyBuildError("optarch is neither an string or a dict %s. This should never happen", optarch) if use_generic: if (self.arch, self.cpu_family) in (self.COMPILER_GENERIC_OPTION or []): @@ -360,10 +358,11 @@ def _set_optimal_architecture(self, default_optarch=None): optarch = self.COMPILER_OPTIMAL_ARCHITECTURE_OPTION[(self.arch, self.cpu_family)] if optarch is not None: - self.log.info("_set_optimal_architecture: using %s as optarch for %s.", optarch, self.arch) + optarch_log_str = optarch or 'no flags' + self.log.info("_set_optimal_architecture: using %s as optarch for %s/%s.", + optarch_log_str, self.arch, self.cpu_family) self.options.options_map['optarch'] = optarch - - if self.options.options_map.get('optarch', None) is None: + elif self.options.options_map.get('optarch', None) is None: optarch_flags_str = "%soptarch flags" % ('', 'generic ')[use_generic] error_msg = "Don't know how to set %s for %s/%s! " % (optarch_flags_str, self.arch, self.cpu_family) error_msg += "Use --optarch='' to override (see " From 5c4fd2f12a8b0f769c5e134e7117d7d8f35ac2a1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 8 Mar 2024 18:19:20 +0100 Subject: [PATCH 131/171] enhance test_deprecated_easyconfig_parameters to cover bugs fixed when there are deprecated easyconfig parameters --- test/framework/easyconfig.py | 46 ++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 141e9b0f77..4a1e1df608 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1733,7 +1733,7 @@ def foo(key): self.assertErrorRegex(EasyBuildError, error_regex, foo, key) def test_deprecated_easyconfig_parameters(self): - """Test handling of replaced easyconfig parameters.""" + """Test handling of deprecated easyconfig parameters.""" os.environ.pop('EASYBUILD_DEPRECATED') easybuild.tools.build_log.CURRENT_VERSION = self.orig_current_version init_config() @@ -1744,10 +1744,13 @@ def test_deprecated_easyconfig_parameters(self): orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS) easyconfig.parser.DEPRECATED_PARAMETERS.update({ 'foobar': ('barfoo', '0.0'), # deprecated since forever - 'foobarbarfoo': ('barfoofoobar', '1000000000'), # won't be actually deprecated for a while + # won't be actually deprecated for a while; + # note that we should map foobarbarfoo to a valid easyconfig parameter here, + # or we'll hit errors when parsing an easyconfig file that uses it + 'foobarbarfoo': ('required_linked_shared_libs', '1000000000'), }) - # copy classes before reloading, so we can restore them (other isinstance checks fail) + # copy classes before reloading, so we can restore them (otherwise isinstance checks fail) orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig) orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS) reload(easyconfig.parser) @@ -1772,6 +1775,35 @@ def foo(key): self.assertEqual(ec[newkey], '123test') self.assertEqual(ec[key], '123test') + variables = { + 'name': 'example', + 'version': '1.2.3', + 'foobar': 'foobar', + 'local_var': 'test', + } + ec = { + 'name': None, + 'version': None, + 'homepage': None, + 'toolchain': None, + } + ec_params, unknown_keys = triage_easyconfig_params(variables, ec) + # deprecated easyconfig parameter 'foobar' is retained as easyconfig parameter; + # only local_var is not retained, since that's a local variable + self.assertEqual(unknown_keys, []) + expected = {'name': 'example', 'version': '1.2.3', 'foobar': 'foobar'} + self.assertEqual(ec_params, expected) + + # try parsing an easyconfig file that defines a deprecated easyconfig parameter + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, read_file(toy_ec)) + write_file(test_ec, "\nfoobarbarfoo = 'foobarbarfoo'", append=True) + + with self.mocked_stdout_stderr(): + ec = EasyConfig(test_ec) + self.assertEqual(ec['required_linked_shared_libs'], 'foobarbarfoo') + easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters reload(easyconfig.parser) reload(easyconfig.easyconfig) @@ -4288,15 +4320,19 @@ def test_triage_easyconfig_params(self): self.assertEqual(sorted(unknown_keys), ['bleh', 'foobar']) # check behaviour when easyconfig parameters that use a name indicating a local variable were defined - ec.update({ + local_vars = { 'x': None, 'local_foo': None, '_foo': None, '_': None, - }) + } + ec.update(local_vars) error = "Found 4 easyconfig parameters that are considered local variables: _, _foo, local_foo, x" self.assertErrorRegex(EasyBuildError, error, triage_easyconfig_params, variables, ec) + for key in local_vars: + del ec[key] + def test_local_vars_detection(self): """Test detection of using unknown easyconfig parameters that are likely local variables.""" From cc615db3136ddabe2882729363c7624337c7924c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Mar 2024 09:59:50 +0100 Subject: [PATCH 132/171] fix code style issues --- easybuild/tools/github.py | 11 +++++++---- test/framework/easyconfig.py | 5 ++--- test/framework/robot.py | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 0cfc181120..c8058d8a8c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -355,7 +355,8 @@ def fetch_latest_commit_sha(repo, account, branch=None, github_user=None, token= return res -def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, account=GITHUB_EB_MAIN, path=None, github_user=None): +def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, account=GITHUB_EB_MAIN, path=None, + github_user=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download @@ -475,7 +476,8 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, extra_ec_paths) + raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", + pr, extra_ec_paths) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) @@ -606,7 +608,8 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, if len(cands) == 1: path = cands[0] else: - raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", commit, extra_ec_paths) + raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", + commit, extra_ec_paths) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit) @@ -640,7 +643,7 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, # if only filename is specified, we need to determine the file path if file == os.path.basename(file): src_path = None - for (dirpath, dirnames, filenames) in os.walk(repo_commit, topdown=True): + for (dirpath, _, filenames) in os.walk(repo_commit, topdown=True): if file in filenames: src_path = os.path.join(dirpath, file) break diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 2cadf46049..baa031c468 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -748,7 +748,7 @@ def test_alt_easyconfig_paths(self): self.assertEqual(tweaked_ecs_path[1], os.path.join(self.test_prefix, 'tweaked_dep_easyconfigs')) self.assertEqual(extra_ecs_path, []) - tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456]) + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123, 456]) self.assertEqual(tweaked_ecs_path, None) self.assertTrue(extra_ecs_path) self.assertTrue(isinstance(extra_ecs_path, list)) @@ -756,8 +756,7 @@ def test_alt_easyconfig_paths(self): self.assertEqual(extra_ecs_path[0], os.path.join(self.test_prefix, 'files_pr123')) self.assertEqual(extra_ecs_path[1], os.path.join(self.test_prefix, 'files_pr456')) - review_pr_path = os.path.join(self.test_prefix, 'review_pr') - tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123,456], + tweaked_ecs_path, extra_ecs_path = alt_easyconfig_paths(self.test_prefix, from_prs=[123, 456], review_pr=789, from_commit='c0ff33') self.assertEqual(tweaked_ecs_path, None) self.assertTrue(extra_ecs_path) diff --git a/test/framework/robot.py b/test/framework/robot.py index 4a23a21500..ccdf9a2b1e 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -722,7 +722,8 @@ def test_github_det_easyconfig_paths_from_commit(self): args = [ os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), # absolute path 'toy-0.0-iter.eb', # relative path, available via robot search path - # commit in which ReFrame-4.3.2.eb was added, see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits + # commit in which ReFrame-4.3.2.eb was added, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18763/commits '--from-commit', commit, 'ReFrame-4.3.2.eb', # easyconfig included in commit, should be resolved via robot search path '--dry-run', From 529cf688c67a9cf45e9404d5239e071acc4bd2f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Mar 2024 10:42:43 +0100 Subject: [PATCH 133/171] make sure that full commit ID is provided to download_repo, and that tarball was actually downloaded + add extra test for download_repo with commit --- easybuild/tools/github.py | 16 ++++++++++++++-- test/framework/github.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c8058d8a8c..cdde9eb748 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -377,6 +377,15 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun mkdir(path, parents=True) if commit: + # make sure that full commit SHA-1 is provided + commit_sha1_regex = re.compile('[0-9a-f]{40}') + if commit_sha1_regex.match(commit): + _log.info("Valid commit SHA-1 provided for downloading %s/%s: %s", account, repo, commit) + else: + error_msg = "Specified commit SHA-1 %s for downloading %s/%s is not valid, " + error_msg += "must be full SHA-1 (40 chars)" + raise EasyBuildError(error_msg, commit, account, repo) + extracted_dir_name = '%s-%s' % (repo, commit) base_name = '%s.tar.gz' % commit latest_commit_sha = commit @@ -401,8 +410,11 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun target_path = os.path.join(path, base_name) _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) - download_file(base_name, url, target_path, forced=True) - _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) + downloaded_path = download_file(base_name, url, target_path, forced=True) + if downloaded_path is None: + raise EasyBuildError("Failed to download tarball for %s/%s commit %s", account, repo, commit) + else: + _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) base_dir = extract_file(target_path, path, forced=True, change_into_dir=False) extracted_path = os.path.join(base_dir, extracted_dir_name) diff --git a/test/framework/github.py b/test/framework/github.py index f011c89b4f..868a653815 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -590,6 +590,34 @@ def test_github_download_repo(self): self.assertTrue(re.match('^[0-9a-f]{40}$', read_file(shafile))) self.assertExists(os.path.join(repodir, 'easybuild', 'easyblocks', '__init__.py')) + def test_github_download_repo_commit(self): + """Test downloading repo at specific commit (which does not require any GitHub token)""" + + # commit bdcc586189fcb3e5a340cddebb50d0e188c63cdc corresponds to easybuild-easyconfigs release v4.9.0 + test_commit = 'bdcc586189fcb3e5a340cddebb50d0e188c63cdc' + gh.download_repo(path=self.test_prefix, commit=test_commit) + repo_path = os.path.join(self.test_prefix, 'easybuilders', 'easybuild-easyconfigs-' + test_commit) + self.assertTrue(os.path.exists(repo_path)) + + setup_py_txt = read_file(os.path.join(repo_path, 'setup.py')) + self.assertTrue("VERSION = '4.9.0'" in setup_py_txt) + + # also check downloading non-default forked repo + test_commit = '434151c3dbf88b2382e8ead8655b4b2c01b92617' + gh.download_repo(path=self.test_prefix, account='boegel', repo='easybuild-framework', commit=test_commit) + repo_path = os.path.join(self.test_prefix, 'boegel', 'easybuild-framework-' + test_commit) + self.assertTrue(os.path.exists(repo_path)) + + release_notes_txt = read_file(os.path.join(repo_path, 'RELEASE_NOTES')) + self.assertTrue("v4.9.0 (30 December 2023)" in release_notes_txt) + + # short commit doesn't work, must be full commit ID + self.assertErrorRegex(EasyBuildError, "Specified commit SHA-1 bdcc586 .* is not valid", gh.download_repo, + path=self.test_prefix, commit='bdcc586') + + self.assertErrorRegex(EasyBuildError, "Failed to download tarball .* commit", gh.download_repo, + path=self.test_prefix, commit='0000000000000000000000000000000000000000') + def test_install_github_token(self): """Test for install_github_token function.""" if self.skip_github_tests: From a6fda46813f8069ba4ba7cb050300b65e15632f1 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 11 Mar 2024 07:30:53 -0400 Subject: [PATCH 134/171] Apply suggestions from code review Flip order of `is_prerelease` and version check for clarity. Co-authored-by: Kenneth Hoste --- easybuild/tools/modules.py | 6 +++--- test/framework/modules.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index ae358e3bbe..3fa735a6b8 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -254,7 +254,7 @@ def set_and_check_version(self): version = LooseVersion(self.version) if self.REQ_VERSION is not None: self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION) - if version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']) or version < self.REQ_VERSION: + if version < self.REQ_VERSION or version.is_prerelease(self.REQ_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s", self.NAME, self.REQ_VERSION, self.version) else: @@ -262,14 +262,14 @@ def set_and_check_version(self): if self.DEPR_VERSION is not None: self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION) - if version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']) or version < self.DEPR_VERSION: + if version < self.DEPR_VERSION or version.is_prerelease(self.DEPR_VERSION, ['rc', '-beta']): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) - if not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']) and self.version > self.MAX_VERSION: + if self.version > self.MAX_VERSION and not version.is_prerelease(self.MAX_VERSION, ['rc', '-beta']): raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s", self.NAME, self.MAX_VERSION, self.version) else: diff --git a/test/framework/modules.py b/test/framework/modules.py index efcad01d5a..dd6d66e70d 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -228,14 +228,14 @@ def test_avail(self): ms = self.modtool.available() version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_prerelease('5.7.5', ['rc']) and version >= '5.7.5': + if isinstance(self.modtool, Lmod) and version >= '5.7.5' and not version.is_prerelease('5.7.5', ['rc']): # with recent versions of Lmod, also the hidden modules are included in the output of 'avail' self.assertEqual(len(ms), TEST_MODULES_COUNT + 3) self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) elif (isinstance(self.modtool, EnvironmentModules) - and not version.is_prerelease('4.6.0', ['-beta'])) and version >= '4.6.0': + and version >= '4.6.0' and not version.is_prerelease('4.6.0', ['-beta'])): # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) self.assertIn('toy/.0.0-deps', ms) @@ -316,7 +316,7 @@ def test_exist(self): avail_mods = self.modtool.available() self.assertIn('Java/1.8.0_181', avail_mods) version = LooseVersion(self.modtool.version) - if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) and version >= '7.0': + if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']): self.assertIn('Java/1.8', avail_mods) self.assertIn('Java/site_default', avail_mods) self.assertIn('JavaAlias', avail_mods) @@ -376,7 +376,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/1.8', 'Core/Java/site_default']), [True, True]) # also check with .modulerc.lua for Lmod 7.8 or newer - if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.8', ['rc']) and version >= '7.8': + if isinstance(self.modtool, Lmod) and version >= '7.8' and not version.is_prerelease('7.8', ['rc']): shutil.move(os.path.join(self.test_prefix, 'Core', 'Java'), java_mod_dir) reset_module_caches() @@ -408,7 +408,7 @@ def test_exist(self): self.assertEqual(self.modtool.exist(['Core/Java/site_default']), [True]) # Test alias in home directory .modulerc - if isinstance(self.modtool, Lmod) and not version.is_prerelease('7.0', ['rc']) or version >= '7.0': + if isinstance(self.modtool, Lmod) and version >= '7.0' and not version.is_prerelease('7.0', ['rc']): # Required or temporary HOME would be in MODULEPATH already self.init_testmods() # Sanity check: Module aliases don't exist yet From 7c71a8792a766604eaf4654dcf7972a3d72e8b02 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 13 Mar 2024 13:00:43 -0400 Subject: [PATCH 135/171] also run unit tests with Python 3.12 + add it to classifiers in setup.py --- .github/workflows/eb_command.yml | 2 +- .github/workflows/linting.yml | 2 +- .github/workflows/unit_tests.yml | 2 ++ setup.py | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index a41b50f572..74245f97b8 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] fail-fast: false steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 93af5bfc29..efd08662b6 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a607ef43df..5698ece42c 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -41,6 +41,8 @@ jobs: modules_tool: ${{needs.setup.outputs.lmod8}} - python: '3.11' modules_tool: ${{needs.setup.outputs.lmod8}} + - python: '3.12' + modules_tool: ${{needs.setup.outputs.lmod8}} # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 diff --git a/setup.py b/setup.py index c7b496867a..b8f8ae77af 100644 --- a/setup.py +++ b/setup.py @@ -116,6 +116,7 @@ def find_rel_test(): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Build Tools", ], platforms="Linux", From 1d4ede3971f9bf17f9c3a74073761fab4b766120 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 13 Mar 2024 13:32:54 -0400 Subject: [PATCH 136/171] Install setuptools for Python 3.12 (only) This (or "pip install build" with "python -m build") is needed for Python 3.12 with "python setup.py sdist" --- .github/workflows/eb_command.yml | 4 ++++ .github/workflows/unit_tests.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index 74245f97b8..ac909b5cd3 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -32,6 +32,10 @@ jobs: # update to latest pip, check version pip install --upgrade pip pip --version + if [[ "${{matrix.python}}" == 3.12 ]]; then + # needed for python setup.py sdist + pip install --upgrade setuptools + fi # for modules tool APT_PKGS="lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev" diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5698ece42c..60fdba402e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -85,6 +85,10 @@ jobs: pip install --upgrade pip pip --version pip install -r requirements.txt + if [[ "${{matrix.python}}" == 3.12 ]]; then + # needed for python setup.py sdist + pip install --upgrade setuptools + fi # git config is required to make actual git commits (cfr. tests for GitRepository) git config --global user.name "Travis CI" git config --global user.email "travis@travis-ci.org" From a7b4fb09bd51b05ab080467c72735a6e6428fb48 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 13 Mar 2024 21:45:47 -0400 Subject: [PATCH 137/171] Don't use "b" for strings with '\uxxxx'. This gives a SyntaxWarning in Python 3.12. In older Pythons it would not warn, but in all cases it gives a literal raw \u (with a backslash) instead of Unicode characters. Python 2.x needed u'\uxxxx' but the u is no longer needed with 3.x. --- test/framework/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 2f16e3978a..539523909f 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -153,7 +153,7 @@ def test_run_cmd(self): # this is constructed to reproduce errors like: # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 # UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018' - for text in [b"foo \xe2 bar", b"foo \u2018 bar"]: + for text in [b"foo \xe2 bar", "foo \u2018 bar"]: test_file = os.path.join(self.test_prefix, 'foo.txt') write_file(test_file, text) cmd = "cat %s" % test_file @@ -182,7 +182,7 @@ def test_run_shell_cmd_basic(self): # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 # UnicodeEncodeError: 'ascii' codec can't encode character u'\u2018' # (such errors are ignored by the 'run' implementation) - for text in [b"foo \xe2 bar", b"foo \u2018 bar"]: + for text in [b"foo \xe2 bar", "foo \u2018 bar"]: test_file = os.path.join(self.test_prefix, 'foo.txt') write_file(test_file, text) cmd = "cat %s" % test_file From 38cc5ead11d98b7f28ddf97adf7f0b09e969fc6e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 14 Mar 2024 13:32:14 -0400 Subject: [PATCH 138/171] Use locale.getpreferredencoding(False) instead of explicit 'utf-8' This fixes Python 3.6 with LC_ALL=C. For Python 3.7+ we always get 'utf-8': https://docs.python.org/3/library/os.html#utf8-mode --- easybuild/tools/run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 2592e198f1..ce2081d58d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -38,6 +38,7 @@ import contextlib import functools import inspect +import locale import os import re import signal @@ -350,8 +351,11 @@ def to_cmd_str(cmd): (stdout, stderr) = proc.communicate(input=stdin) # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) - output = stdout.decode('utf-8', 'ignore') - stderr = stderr.decode('utf-8', 'ignore') if split_stderr else None + # getpreferredencoding normally gives 'utf-8' but can be ASCII (ANSI_X3.4-1968) + # for Python 3.6 and older with LC_ALL=C + encoding = locale.getpreferredencoding(False) + output = stdout.decode(encoding, 'ignore') + stderr = stderr.decode(encoding, 'ignore') if split_stderr else None # store command output to temporary file(s) if output_file: From 3e117fec6efb65f26b459dd272f263f9b76e144e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 16 Mar 2024 10:47:35 +0100 Subject: [PATCH 139/171] add tests for fetch_files_from_commit and fetch_easyconfigs_from_commit --- easybuild/tools/github.py | 22 +++++++++----- test/framework/github.py | 64 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index cdde9eb748..2f67285e1b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -377,12 +377,12 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun mkdir(path, parents=True) if commit: - # make sure that full commit SHA-1 is provided + # make sure that full commit SHA is provided commit_sha1_regex = re.compile('[0-9a-f]{40}') if commit_sha1_regex.match(commit): - _log.info("Valid commit SHA-1 provided for downloading %s/%s: %s", account, repo, commit) + _log.info("Valid commit SHA provided for downloading %s/%s: %s", account, repo, commit) else: - error_msg = "Specified commit SHA-1 %s for downloading %s/%s is not valid, " + error_msg = r"Specified commit SHA %s for downloading %s/%s is not valid, " error_msg += "must be full SHA-1 (40 chars)" raise EasyBuildError(error_msg, commit, account, repo) @@ -622,6 +622,8 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, else: raise EasyBuildError("Failed to isolate path for commit %s from list of commit paths: %s", commit, extra_ec_paths) + else: + path = os.path.join(tempfile.gettempdir(), 'ecs_commit_' + commit) elif github_repo == GITHUB_EASYBLOCKS_REPO: path = os.path.join(tempfile.gettempdir(), 'ebs_commit_' + commit) @@ -633,12 +635,14 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff') diff_fn = os.path.basename(diff_url) diff_filepath = os.path.join(path, diff_fn) - download_file(diff_fn, diff_url, diff_filepath, forced=True) - diff_txt = read_file(diff_filepath) - _log.debug("Diff for commit %s:\n%s", commit, diff_txt) + if download_file(diff_fn, diff_url, diff_filepath, forced=True): + diff_txt = read_file(diff_filepath) + _log.debug("Diff for commit %s:\n%s", commit, diff_txt) - files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True) - _log.debug("List of patched files for commit %s: %s", commit, files) + files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True) + _log.debug("List of patched files for commit %s: %s", commit, files) + else: + raise EasyBuildError("Failed to download diff for commit %s of %s/%s", commit, github_account, github_repo) # download tarball for specific commit repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account) @@ -647,6 +651,8 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, files_subdir = 'easybuild/easyconfigs/' elif github_repo == GITHUB_EASYBLOCKS_REPO: files_subdir = 'easybuild/easyblocks/' + else: + raise EasyBuildError("Unknown repo: %s" % github_repo) # copy specified files to directory where they're expected to be found file_paths = [] diff --git a/test/framework/github.py b/test/framework/github.py index 868a653815..4bcf9b8987 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -50,7 +50,8 @@ from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_MERGEABLE_STATE_CLEAN from easybuild.tools.github import VALID_CLOSE_PR_REASONS -from easybuild.tools.github import det_pr_title, is_patch_for, pick_default_branch +from easybuild.tools.github import det_pr_title, fetch_easyconfigs_from_commit, fetch_files_from_commit +from easybuild.tools.github import is_patch_for, pick_default_branch from easybuild.tools.testing import create_test_report, post_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -536,6 +537,65 @@ def test_github_fetch_files_from_pr_cache(self): res = gh.fetch_easyblocks_from_pr(12345, tmpdir) self.assertEqual(sorted(pr12345_files), sorted(res)) + def test_fetch_files_from_commit(self): + """Test fetch_files_from_commit function.""" + + # easyconfigs commit to add EasyBuild-4.8.2.eb (also test short commit, should work) + test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' + + # without specifying any files/repo, default is to use easybuilders/easybuilld-easyconfigs + # and determine which files were changed in the commit + res = fetch_files_from_commit(test_commit) + self.assertEqual(len(res), 1) + ec_path = res[0] + expected_path = 'ecs_commit_7c83a553950c233943c7b0189762f8c05cfea852/e/EasyBuild/EasyBuild-4.8.2.eb' + self.assertTrue(ec_path.endswith(expected_path)) + self.assertTrue(os.path.exists(ec_path)) + self.assertIn("version = '4.8.2'", read_file(ec_path)) + + # also test downloading a specific file from easyblocks repo + # commit that enables use_pip & co in PythonPackage easyblock + test_commit = 'd6f0cd7b586108e40f7cf1f1054bb07e16718caf' + res = fetch_files_from_commit(test_commit, files=['pythonpackage.py'], + github_account='easybuilders', github_repo='easybuild-easyblocks') + self.assertEqual(len(res), 1) + self.assertIn("'use_pip': [True,", read_file(res[0])) + + # test downloading with short commit, download_repo currently enforces using long commit + error_pattern = r"Specified commit SHA 7c83a55 for downloading easybuilders/easybuild-easyconfigs " + error_pattern += r"is not valid, must be full SHA-1 \(40 chars\)" + self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, '7c83a55') + + # test downloading of non-existing commit + error_pattern = r"Failed to download diff for commit c0ff33c0ff33 of easybuilders/easybuild-easyconfigs" + self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, 'c0ff33c0ff33') + + def test_fetch_easyconfigs_from_commit(self): + """Test fetch_easyconfigs_from_commit function.""" + + # commit in which easyconfigs for PyTables 3.9.2 + dependencies were added + test_commit = '6515b44cd84a20fe7876cb4bdaf3c0080e688566' + + # without specifying any files/repo, default is to determine which files were changed in the commit + res = fetch_easyconfigs_from_commit(test_commit) + self.assertEqual(len(res), 5) + expected_ec_filenames = ['Blosc-1.21.5-GCCcore-13.2.0.eb', 'Blosc2-2.13.2-GCCcore-13.2.0.eb', + 'PyTables-3.9.2-foss-2023b.eb', 'PyTables-3.9.2_fix-find-blosc2-dep.patch', + 'py-cpuinfo-9.0.0-GCCcore-13.2.0.eb'] + self.assertEqual(sorted([os.path.basename(f) for f in res]), expected_ec_filenames) + for ec_path in res: + self.assertTrue(os.path.exists(ec_path)) + if ec_path.endswith('.eb'): + self.assertIn("version =", read_file(ec_path)) + else: + self.assertTrue(ec_path.endswith('.patch')) + + # merge commit for release of EasyBuild v4.9.0 + test_commit = 'bdcc586189fcb3e5a340cddebb50d0e188c63cdc' + res = fetch_easyconfigs_from_commit(test_commit, files=['RELEASE_NOTES'], path=self.test_prefix) + self.assertEqual(len(res), 1) + self.assertIn("v4.9.0 (30 December 2023)", read_file(res[0])) + def test_github_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" if self.skip_github_tests: @@ -612,7 +672,7 @@ def test_github_download_repo_commit(self): self.assertTrue("v4.9.0 (30 December 2023)" in release_notes_txt) # short commit doesn't work, must be full commit ID - self.assertErrorRegex(EasyBuildError, "Specified commit SHA-1 bdcc586 .* is not valid", gh.download_repo, + self.assertErrorRegex(EasyBuildError, "Specified commit SHA bdcc586 .* is not valid", gh.download_repo, path=self.test_prefix, commit='bdcc586') self.assertErrorRegex(EasyBuildError, "Failed to download tarball .* commit", gh.download_repo, From b6e301898c364aeee1840bb066447cec7ccb18a1 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 18 Mar 2024 07:34:11 -0400 Subject: [PATCH 140/171] Explicitly check if `python -c "import distutils"` is possible This makes the check more general and compatible with later versions of Python than just 3.12. --- .github/workflows/eb_command.yml | 4 ++-- .github/workflows/unit_tests.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index ac909b5cd3..ec99594a86 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -32,8 +32,8 @@ jobs: # update to latest pip, check version pip install --upgrade pip pip --version - if [[ "${{matrix.python}}" == 3.12 ]]; then - # needed for python setup.py sdist + if ! python -c "import distutils" 2> /dev/null; then + # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist pip install --upgrade setuptools fi diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 60fdba402e..ce739ff46d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -85,8 +85,8 @@ jobs: pip install --upgrade pip pip --version pip install -r requirements.txt - if [[ "${{matrix.python}}" == 3.12 ]]; then - # needed for python setup.py sdist + if ! python -c "import distutils" 2> /dev/null; then + # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist pip install --upgrade setuptools fi # git config is required to make actual git commits (cfr. tests for GitRepository) From f5489bf2780256291a785e22bbdfe2c05c638407 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 24 Mar 2024 11:38:01 +0100 Subject: [PATCH 141/171] add test for --from-commit --- test/framework/options.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 26daa0cc7a..6cd46de021 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2011,6 +2011,44 @@ def test_github_from_pr_x(self): except URLError as err: print("Ignoring URLError '%s' in test_from_pr_x" % err) + def test_from_commit(self): + """Test for --from-commit.""" + # note: --from-commit does not involve using GitHub API, so no GitHub token required + + # easyconfigs commit to add EasyBuild-4.8.2.eb + test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + tmpdir = tempfile.mkdtemp() + args = [ + '--from-commit=%s' % test_commit, + '--dry-run', + '--tmpdir=%s' % tmpdir, + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'EasyBuild/4.8.2'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + path = '.*%s' % os.path.dirname(path_prefix) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit) + regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError as err: + print("Ignoring URLError '%s' in test_from_pr" % err) + shutil.rmtree(tmpdir) + def test_no_such_software(self): """Test using no arguments.""" From fd7b9f9fcadb860ab0f26e11f10eee3ef14e1789 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 27 Mar 2024 14:18:24 +0100 Subject: [PATCH 142/171] add --short option to findUpdatedEcs This is useful if you want to find & copy the names of ECs to rebuild. Also fixed indentation (4 spaces consistently) and use a unified diff. --- easybuild/scripts/findUpdatedEcs.sh | 58 ++++++++++++++++------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh index d6631bfe0b..66b7e34bd8 100755 --- a/easybuild/scripts/findUpdatedEcs.sh +++ b/easybuild/scripts/findUpdatedEcs.sh @@ -33,51 +33,57 @@ function checkModule { first_letter=${ec_filename:0:1} letterPath=$easyconfigFolder/${first_letter,,} if [[ -d "$letterPath" ]]; then - ec_new="$(find "$letterPath" -type f -name "$ec_filename")" + ec_new="$(find "$letterPath" -type f -name "$ec_filename")" else - ec_new= + ec_new= fi # Fallback if not found [[ -n "$ec_new" ]] || ec_new="$(find "$easyconfigFolder" -type f -name "$ec_filename")" if [[ -z "$ec_new" ]]; then - printError "=== Did not find new EC $ec_filename" + printError "=== Did not find new EC $ec_filename" elif [[ ! -e "$ec_new" ]]; then printError "=== Found multiple new ECs: $ec_new" - elif ! out=$(diff "$ec_installed" "$ec_new"); then - echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}" - if ((showDiff == 1)); then - echo "$out" + elif ! out=$(diff -u "$ec_installed" "$ec_new"); then + if ((short == 1)); then + basename "$ec_installed" + else + echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}" + if ((showDiff == 1)); then + echo "$out" + fi fi fi } ecDefaultFolder= if path=$(which eb 2>/dev/null); then - path=$(dirname "$path") - for p in "$path" "$(dirname "$path")"; do - if [ -d "$p/easybuild/easyconfigs" ]; then - ecDefaultFolder=$p - break - fi - done + path=$(dirname "$path") + for p in "$path" "$(dirname "$path")"; do + if [ -d "$p/easybuild/easyconfigs" ]; then + ecDefaultFolder=$p + break + fi + done fi function usage { - echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" - echo - echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed." - echo "Can either check the currently loaded modules or all modules installed in a specific location" - echo - echo "--verbose Verbose status output while checking" - echo "--loaded Check only currently loaded modules" - echo "--diff Show diff of changed module files" - echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries" - echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}" - exit 0 + echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER" + echo + echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed." + echo "Can either check the currently loaded modules or all modules installed in a specific location" + echo + echo "--verbose Verbose status output while checking" + echo "--loaded Check only currently loaded modules" + echo "--short Only show filename of changed ECs" + echo "--diff Show diff of changed module files" + echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries" + echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}" + exit 0 } checkLoadedModules=0 showDiff=0 +short=0 modulesFolder="" easyconfigFolder=$ecDefaultFolder @@ -89,6 +95,8 @@ while [[ $# -gt 0 ]]; do verbose=1;; -d|--diff) showDiff=1;; + -s|--short) + short=1;; -l|--loaded) checkLoadedModules=1;; -m|--modules) From 162aad0e7f1b8a27dd22a6e6bb198ef5af9367ee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Mar 2024 21:36:13 +0100 Subject: [PATCH 143/171] implement support for --include-easyblocks-from-commit --- easybuild/tools/config.py | 1 + easybuild/tools/github.py | 7 ++- easybuild/tools/options.py | 110 +++++++++++++++++++++++-------------- easybuild/tools/testing.py | 8 +++ 4 files changed, 85 insertions(+), 41 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ebc4f4a7d1..21df1f1e91 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -230,6 +230,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'http_header_fields_urlpat', 'hooks', 'ignore_dirs', + 'include_easyblocks_from_commit', 'insecure_download', 'job_backend_config', 'job_cores', diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 2f67285e1b..eda237fca1 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -590,7 +590,7 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi def fetch_easyblocks_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" + """Fetch patched easyblocks for a particular PR.""" return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) @@ -685,6 +685,11 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, return file_paths +def fetch_easyblocks_from_commit(commit, files=None, path=None): + """Fetch easyblocks from a specified commit.""" + return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYBLOCKS_REPO) + + def fetch_easyconfigs_from_commit(commit, files=None, path=None): """Fetch specified easyconfig files from a specific commit.""" return fetch_files_from_commit(commit, files=files, path=path, github_repo=GITHUB_EASYCONFIGS_REPO) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index f99a5ac284..f9093ecc94 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -89,7 +89,7 @@ from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS -from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token +from easybuild.tools.github import fetch_easyblocks_from_commit, fetch_easyblocks_from_pr, fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains from easybuild.tools.job.backend import avail_job_backends @@ -702,11 +702,13 @@ def github_options(self): 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), - 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'PR#'}), + 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'commit_SHA'}), 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), 'github-org': ("GitHub organization", str, 'store', None), + 'include-easyblocks-from-commit': ("Include easyblocks from specified commit", 'str', 'store', None, + {'metavar': 'commit_SHA'}), 'include-easyblocks-from-pr': ("Include easyblocks from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'install-github-token': ("Install GitHub token (requires --github-user)", None, 'store_true', False), @@ -1225,8 +1227,9 @@ def _postprocess_list_avail(self): if self.options.avail_easyconfig_licenses: msg += avail_easyconfig_licenses(self.options.output_format) - # dump available easyblocks (unless including easyblocks from pr, in which case it will be done later) - if self.options.list_easyblocks and not self.options.include_easyblocks_from_pr: + # dump available easyblocks (unless including easyblocks from commit or PR, in which case it will be done later) + easyblocks_from = self.options.include_easyblocks_from_commit or self.options.include_easyblocks_from_pr + if self.options.list_easyblocks and not easyblocks_from: msg += list_easyblocks(self.options.list_easyblocks, self.options.output_format) # dump known toolchains @@ -1270,7 +1273,7 @@ def _postprocess_list_avail(self): print(msg) # cleanup tmpdir and exit - if not self.options.include_easyblocks_from_pr: + if not (self.options.include_easyblocks_from_commit or self.options.include_easyblocks_from_pr): cleanup_and_exit(self.tmpdir) def avail_repositories(self): @@ -1524,6 +1527,67 @@ def check_root_usage(allow_use_as_root=False): raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") +def handle_include_easyblocks_from(options, log): + """ + Handle --include-easyblocks-from-pr and --include-easyblocks-from-commit + """ + def check_included_multiple(included_easyblocks_from, source): + """Check whether easyblock is being included multiple times""" + included_multiple = included_easyblocks_from & included_easyblocks + if included_multiple: + warning_msg = "One or more easyblocks included from multiple locations: %s " \ + % ', '.join(included_multiple) + warning_msg += "(the one(s) from %s will be used)" % source + print_warning(warning_msg) + + if options.include_easyblocks_from_pr or options.include_easyblocks_from_commit: + + if options.include_easyblocks: + # check if you are including the same easyblock twice + included_paths = expand_glob_paths(options.include_easyblocks) + included_easyblocks = set([os.path.basename(eb) for eb in included_paths]) + + if options.include_easyblocks_from_pr: + try: + easyblock_prs = [int(x) for x in options.include_easyblocks_from_pr] + except ValueError: + raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") + + for easyblock_pr in easyblock_prs: + easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr) + included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr]) + + if options.include_easyblocks: + check_included_multiple(included_from_pr, "PR #%s" % easyblock_pr) + + included_easyblocks |= included_from_pr + + for easyblock in included_from_pr: + print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log) + + include_easyblocks(options.tmpdir, easyblocks_from_pr) + + easyblock_commit = options.include_easyblocks_from_commit + if easyblock_commit: + easyblocks_from_commit = fetch_easyblocks_from_commit(easyblock_commit) + included_from_commit = set([os.path.basename(eb) for eb in easyblocks_from_commit]) + + if options.include_easyblocks: + check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) + + for easyblock in included_from_commit: + print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log) + + include_easyblocks(options.tmpdir, easyblocks_from_commit) + + if options.list_easyblocks: + msg = list_easyblocks(options.list_easyblocks, options.output_format) + if options.unittest_file: + log.info(msg) + else: + print(msg) + cleanup_and_exit(tmpdir) + def set_up_configuration(args=None, logfile=None, testing=False, silent=False, reconfigure=False): """ @@ -1633,41 +1697,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r init_build_options(build_options=build_options, cmdline_options=options) # done here instead of in _postprocess_include because github integration requires build_options to be initialized - if eb_go.options.include_easyblocks_from_pr: - try: - easyblock_prs = [int(x) for x in eb_go.options.include_easyblocks_from_pr] - except ValueError: - raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") - - if eb_go.options.include_easyblocks: - # check if you are including the same easyblock twice - included_paths = expand_glob_paths(eb_go.options.include_easyblocks) - included_from_file = set([os.path.basename(eb) for eb in included_paths]) - - for easyblock_pr in easyblock_prs: - easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr) - included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr]) - - if eb_go.options.include_easyblocks: - included_twice = included_from_pr & included_from_file - if included_twice: - warning_msg = "One or more easyblocks included from multiple locations: %s " \ - % ', '.join(included_twice) - warning_msg += "(the one(s) from PR #%s will be used)" % easyblock_pr - print_warning(warning_msg) - - for easyblock in included_from_pr: - print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log) - - include_easyblocks(eb_go.options.tmpdir, easyblocks_from_pr) - - if eb_go.options.list_easyblocks: - msg = list_easyblocks(eb_go.options.list_easyblocks, eb_go.options.output_format) - if eb_go.options.unittest_file: - log.info(msg) - else: - print(msg) - cleanup_and_exit(tmpdir) + handle_include_easyblocks_from(eb_go.options, log) check_python_version() diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index ffd8ce580b..a58e539c23 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -336,6 +336,14 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state, comment_lines = ["Test report by @%s" % github_user] + if build_option('include_easyblocks_from_commit'): + if repo_type == GITHUB_EASYCONFIGS_REPO: + easyblocks_commit = build_option('include_easyblocks_from_commit') + url = 'https://github.com/%s/%s/commit/%s' % (pr_target_account, GITHUB_EASYBLOCKS_REPO, easyblocks_commit) + comment_lines.append("Using easyblocks from %s" % url) + else: + raise EasyBuildError("Don't know how to submit test reports to repo %s.", repo_type) + if build_option('include_easyblocks_from_pr'): if repo_type == GITHUB_EASYCONFIGS_REPO: easyblocks_pr_nrs = [int(x) for x in build_option('include_easyblocks_from_pr')] From 44f94c22fbfbcd230d83ae133b509ee2a26cb9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Mar 2024 09:54:01 +0100 Subject: [PATCH 144/171] define flags for --optarch=GENERIC on RISC-V --- easybuild/toolchains/compiler/clang.py | 1 + easybuild/toolchains/compiler/gcc.py | 1 + 2 files changed, 2 insertions(+) diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 303f0ed0e4..4ab46abde7 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -98,6 +98,7 @@ class Clang(Compiler): } # used with --optarch=GENERIC COMPILER_GENERIC_OPTION = { + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 3fba8d3954..84a849d596 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -94,6 +94,7 @@ class Gcc(Compiler): (systemtools.AARCH64, systemtools.ARM): 'mcpu=generic', # implies -march=armv8-a and -mtune=generic (systemtools.POWER, systemtools.POWER): 'mcpu=powerpc64', # no support for -march on POWER (systemtools.POWER, systemtools.POWER_LE): 'mcpu=powerpc64le', # no support for -march on POWER + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } From ab93030fb9831426a1ac6515623cf68c686852ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Mar 2024 10:23:56 +0100 Subject: [PATCH 145/171] add additional space, shorten line --- easybuild/toolchains/compiler/clang.py | 2 +- easybuild/toolchains/compiler/gcc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 4ab46abde7..8c93053932 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -98,7 +98,7 @@ class Clang(Compiler): } # used with --optarch=GENERIC COMPILER_GENERIC_OPTION = { - (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 84a849d596..fd20846592 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -94,7 +94,7 @@ class Gcc(Compiler): (systemtools.AARCH64, systemtools.ARM): 'mcpu=generic', # implies -march=armv8-a and -mtune=generic (systemtools.POWER, systemtools.POWER): 'mcpu=powerpc64', # no support for -march on POWER (systemtools.POWER, systemtools.POWER_LE): 'mcpu=powerpc64le', # no support for -march on POWER - (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # the default for -mabi is system-dependent + (systemtools.RISCV64, systemtools.RISCV): 'march=rv64gc -mabi=lp64d', # default for -mabi is system-dependent (systemtools.X86_64, systemtools.AMD): 'march=x86-64 -mtune=generic', (systemtools.X86_64, systemtools.INTEL): 'march=x86-64 -mtune=generic', } From f2ed0ff9f9184983579f7815ce3be473536b34e7 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 31 Mar 2024 11:06:54 +0100 Subject: [PATCH 146/171] hide readelf output --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index aa77793929..9f7ac535a7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3187,7 +3187,7 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # check whether RPATH section in 'readelf -d' output is there if check_readelf_rpath: fail_msg = None - res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False) + res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False, hidden=True) if res.exit_code: fail_msg = f"Failed to run 'readelf -d {path}': {res.output}" elif not readelf_rpath_regex.search(res.output): From f8c683b6140c40dd47b74b0291b2165a89c44ad6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sun, 31 Mar 2024 11:45:34 +0100 Subject: [PATCH 147/171] switch from to `bash` in tests --- test/framework/filetools.py | 2 +- test/framework/systemtools.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 91afdf79f1..db85575cdb 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -924,7 +924,7 @@ def test_is_binary(self): self.assertTrue(ft.is_binary(b'\00')) self.assertTrue(ft.is_binary(b"File is binary when it includes \00 somewhere")) - self.assertTrue(ft.is_binary(ft.read_file('/bin/ls', mode='rb'))) + self.assertTrue(ft.is_binary(ft.read_file('/bin/bash', mode='rb'))) def test_det_patched_files(self): """Test det_patched_files function.""" diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 6d8395a5fc..68a81c814f 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1033,15 +1033,15 @@ def test_check_linked_shared_libs(self): self.assertEqual(check_linked_shared_libs(txt_path), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path), None) - bin_ls_path = which('ls') + bin_bash_path = which('bash') os_type = get_os_type() if os_type == LINUX: with self.mocked_stdout_stderr(): - res = run_shell_cmd("ldd %s" % bin_ls_path) + res = run_shell_cmd("ldd %s" % bin_bash_path) elif os_type == DARWIN: with self.mocked_stdout_stderr(): - res = run_shell_cmd("otool -L %s" % bin_ls_path) + res = run_shell_cmd("otool -L %s" % bin_bash_path) else: raise EasyBuildError("Unknown OS type: %s" % os_type) @@ -1061,7 +1061,7 @@ def test_check_linked_shared_libs(self): self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) - for path in (bin_ls_path, lib_path): + for path in (bin_bash_path, lib_path): # path may not exist, especially for library paths obtained via 'otool -L' on macOS if os.path.exists(path): error_msg = "Check on linked libs should pass for %s with %s" % (path, pattern_named_args) @@ -1078,7 +1078,7 @@ def test_check_linked_shared_libs(self): self.assertEqual(check_linked_shared_libs(self.test_prefix, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(txt_path, **pattern_named_args), None) self.assertEqual(check_linked_shared_libs(broken_symlink_path, **pattern_named_args), None) - for path in (bin_ls_path, lib_path): + for path in (bin_bash_path, lib_path): error_msg = "Check on linked libs should fail for %s with %s" % (path, pattern_named_args) self.assertFalse(check_linked_shared_libs(path, **pattern_named_args), error_msg) From ba306a4966d24f3e89acc31e87bbe48ad83fe638 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Apr 2024 21:30:54 +0200 Subject: [PATCH 148/171] fix use of 'tmpdir' in handle_include_easyblocks_from + trivial style fixes --- easybuild/tools/options.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index f9093ecc94..91bede9553 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -702,7 +702,8 @@ def github_options(self): 'check-style': ("Run a style check on the given easyconfigs", None, 'store_true', False), 'cleanup-easyconfigs': ("Clean up easyconfig files for pull request", None, 'store_true', True), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), - 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', None, {'metavar': 'commit_SHA'}), + 'from-commit': ("Obtain easyconfigs from specified commit", 'str', 'store', + None, {'metavar': 'commit_SHA'}), 'from-pr': ("Obtain easyconfigs from specified PR", 'strlist', 'store', [], {'metavar': 'PR#'}), 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), @@ -1527,6 +1528,7 @@ def check_root_usage(allow_use_as_root=False): raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") + def handle_include_easyblocks_from(options, log): """ Handle --include-easyblocks-from-pr and --include-easyblocks-from-commit @@ -1551,7 +1553,7 @@ def check_included_multiple(included_easyblocks_from, source): try: easyblock_prs = [int(x) for x in options.include_easyblocks_from_pr] except ValueError: - raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.") + raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s") for easyblock_pr in easyblock_prs: easyblocks_from_pr = fetch_easyblocks_from_pr(easyblock_pr) @@ -1573,7 +1575,7 @@ def check_included_multiple(included_easyblocks_from, source): included_from_commit = set([os.path.basename(eb) for eb in easyblocks_from_commit]) if options.include_easyblocks: - check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) + check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) for easyblock in included_from_commit: print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log) @@ -1586,6 +1588,8 @@ def check_included_multiple(included_easyblocks_from, source): log.info(msg) else: print(msg) + # tmpdir is set by option parser via set_tmpdir function + tmpdir = tempfile.gettempdir() cleanup_and_exit(tmpdir) From 799989a202a59e29d71dd4d825afbf26e7d96a77 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Apr 2024 21:56:50 +0200 Subject: [PATCH 149/171] add test for --include-easyblocks-from-commit --- test/framework/options.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 6cd46de021..3b9a46d0f5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2028,6 +2028,7 @@ def test_from_commit(self): '--tmpdir=%s' % tmpdir, ] try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ (tmpdir, 'EasyBuild/4.8.2'), @@ -2046,7 +2047,43 @@ def test_from_commit(self): regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) except URLError as err: - print("Ignoring URLError '%s' in test_from_pr" % err) + print("Ignoring URLError '%s' in test_from_commit" % err) + shutil.rmtree(tmpdir) + + def test_include_easyblocks_from_commit(self): + """Test for --include-easyblocks-from-commit.""" + # note: --include-easyblocks-from-commit does not involve using GitHub API, so no GitHub token required + + # easyblocks commit only touching ConfigureMake easyblock + test_commit = '6005d37a1ed2b130a18b5bd525df810f19ba3bbd' + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + tmpdir = tempfile.mkdtemp() + args = [ + '--include-easyblocks-from-commit=%s' % test_commit, + '--dry-run', + '--tmpdir=%s' % tmpdir, + 'toy-0.0.eb', # test easyconfig + ] + try: + self.mock_stdout(True) + self.mock_stderr(True) + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + pattern = "== easyblock configuremake.py included from comit %s" % test_commit + self.assertEqual(stderr, '') + self.assertIn(pattern, stdout) + + regex = re.compile(r"^ \* \[.\] .*/toy-0.0.eb \(module: toy/0.0\)$", re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + except URLError as err: + print("Ignoring URLError '%s' in test_include_easyblocks_from_commit" % err) shutil.rmtree(tmpdir) def test_no_such_software(self): From de5e2350102eaa4792db4edf7bf6098605e514f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Apr 2024 22:32:34 +0200 Subject: [PATCH 150/171] fix combination of --copy-ec and --from-commit --- easybuild/framework/easyconfig/tools.py | 41 +++++++++++++++- easybuild/main.py | 2 +- test/framework/options.py | 65 ++++++++++++++++++++++++- 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 711e14d29e..7737d72860 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -62,7 +62,7 @@ from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data -from easybuild.tools.github import fetch_files_from_pr +from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -791,7 +791,7 @@ def avail_easyblocks(): return easyblocks -def det_copy_ec_specs(orig_paths, from_pr): +def det_copy_ec_specs(orig_paths, from_pr=None, from_commit=None): """Determine list of paths + target directory for --copy-ec.""" if from_pr is not None and not isinstance(from_pr, list): @@ -855,4 +855,41 @@ def det_copy_ec_specs(orig_paths, from_pr): elif pr_matches: raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches) + # consider --from-commit (only if --from-pr was not used) + elif from_commit: + tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_commit_%s' % from_commit) + commit_paths = fetch_files_from_commit(from_commit, path=tmpdir) + + # assume that files need to be copied to current working directory for now + target_path = os.getcwd() + + if orig_paths: + last_path = orig_paths[-1] + + # check files touched by commit and see if the target directory for --copy-ec + # corresponds to the name of one of these files; + # if so we should copy the specified file(s) to the current working directory, + # since interpreting the last argument as target location is very unlikely to be correct in this case + commit_filenames = [os.path.basename(p) for p in commit_paths] + if last_path in commit_filenames: + paths = orig_paths[:] + else: + target_path = last_path + # exclude last argument that is used as target location + paths = orig_paths[:-1] + + # if list of files to copy is empty at this point, + # we simply copy *all* files touched by the PR + if not paths: + paths = commit_paths + + # replace path for files touched by commit (no need to worry about others) + for idx, path in enumerate(paths): + filename = os.path.basename(path) + commit_matches = [x for x in commit_paths if os.path.basename(x) == filename] + if len(commit_matches) == 1: + paths[idx] = commit_matches[0] + elif commit_matches: + raise EasyBuildError("Found multiple paths for %s in commit: %s", filename, commit_matches) + return paths, target_path diff --git a/easybuild/main.py b/easybuild/main.py index 85501f57f0..4ca7bcbb95 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -329,7 +329,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session if options.copy_ec: # figure out list of files to copy + target location (taking into account --from-pr) - eb_args, target_path = det_copy_ec_specs(eb_args, from_pr_list) + eb_args, target_path = det_copy_ec_specs(eb_args, from_pr=from_pr_list, from_commit=options.from_commit) categorized_paths = categorize_files_by_type(eb_args) diff --git a/test/framework/options.py b/test/framework/options.py index 3b9a46d0f5..623aff63f6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1422,6 +1422,70 @@ def test_github_copy_ec_from_pr(self): self.assertIn("name = 'ExifTool'", read_file(test_ec)) remove_file(test_ec) + def test_copy_ec_from_commit(self): + """Test combination of --copy-ec with --from-commit.""" + # note: --from-commit does not involve using GitHub API, so no GitHub token required + + # using easyconfigs commit to add EasyBuild-4.8.2.eb + test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' + + test_dir = os.path.join(self.test_prefix, 'from_commit') + mkdir(test_dir, parents=True) + args = ['--copy-ec', '--from-commit=%s' % test_commit, test_dir] + try: + stdout = self.mocked_main(args) + except URLError as err: + print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err) + + pattern = "_%s/e/EasyBuild/EasyBuild-4.8.2.eb copied to " % test_commit + self.assertIn(pattern, stdout) + copied_ecs = os.listdir(test_dir) + self.assertEqual(copied_ecs, ['EasyBuild-4.8.2.eb']) + + # cleanup + remove_dir(test_dir) + mkdir(test_dir) + + # test again, using extra argument (name of file to copy), without specifying target directory + # (should copy to current directory) + cwd = change_dir(test_dir) + args = ['--copy-ec', '--from-commit=%s' % test_commit, "EasyBuild-4.8.2.eb"] + try: + stdout = self.mocked_main(args) + except URLError as err: + print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err) + + self.assertIn(pattern, stdout) + copied_ecs = os.listdir(test_dir) + self.assertEqual(copied_ecs, ['EasyBuild-4.8.2.eb']) + + # cleanup + change_dir(cwd) + remove_dir(test_dir) + mkdir(test_dir) + + # test with commit that touches a bunch of easyconfigs + test_commit = '49c887397b1a948e1909fc24bc905fdc1ad38388' + expected_ecs = [ + 'gompi-2023b.eb', + 'gfbf-2023b.eb', + 'ScaLAPACK-2.2.0-gompi-2023b-fb.eb', + 'foss-2023b.eb', + 'HPL-2.3-foss-2023b.eb', + 'FFTW.MPI-3.3.10-gompi-2023b.eb', + 'SciPy-bundle-2023.11-gfbf-2023b.eb', + 'OSU-Micro-Benchmarks-7.2-gompi-2023b.eb', + ] + args = ['--copy-ec', '--from-commit=%s' % test_commit, test_dir] + try: + stdout = self.mocked_main(args) + except URLError as err: + print("Ignoring URLError '%s' in test_copy_ec_from_commit" % err) + + copied_ecs = os.listdir(test_dir) + for ec in expected_ecs: + self.assertIn(ec, copied_ecs) + def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -2028,7 +2092,6 @@ def test_from_commit(self): '--tmpdir=%s' % tmpdir, ] try: - outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) modules = [ (tmpdir, 'EasyBuild/4.8.2'), From 90c4939396e1d8e8a0b9d5649dddca721e3db346 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 09:11:09 +0200 Subject: [PATCH 151/171] use different commit for testing --include-easyblocks-from-commit + clean up imported easyblock --- test/framework/options.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 623aff63f6..8b578d74da 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2113,12 +2113,15 @@ def test_from_commit(self): print("Ignoring URLError '%s' in test_from_commit" % err) shutil.rmtree(tmpdir) - def test_include_easyblocks_from_commit(self): + # must be run after test for --list-easyblocks, hence the '_xxx_' + # cleaning up the imported easyblocks is quite difficult... + def test_xxx_include_easyblocks_from_commit(self): """Test for --include-easyblocks-from-commit.""" # note: --include-easyblocks-from-commit does not involve using GitHub API, so no GitHub token required - # easyblocks commit only touching ConfigureMake easyblock - test_commit = '6005d37a1ed2b130a18b5bd525df810f19ba3bbd' + orig_local_sys_path = sys.path[:] + # easyblocks commit only touching Binary easyblock + test_commit = '94d28c556947bd96d0978df775b15a50a4600c6f' fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) @@ -2138,7 +2141,16 @@ def test_include_easyblocks_from_commit(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) - pattern = "== easyblock configuremake.py included from comit %s" % test_commit + + # 'undo' import of foo easyblock + del sys.modules['easybuild.easyblocks.generic.binary'] + sys.path[:] = orig_local_sys_path + import easybuild.easyblocks + reload(easybuild.easyblocks) + import easybuild.easyblocks.generic + reload(easybuild.easyblocks.generic) + + pattern = "== easyblock binary.py included from comit %s" % test_commit self.assertEqual(stderr, '') self.assertIn(pattern, stdout) From 818e65a6e13981a3bd961adf2107d3ad5ad48a7e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 09:14:28 +0200 Subject: [PATCH 152/171] update copyright lines to 2024 --- easybuild/__init__.py | 2 +- easybuild/base/exceptions.py | 2 +- easybuild/base/fancylogger.py | 2 +- easybuild/base/generaloption.py | 2 +- easybuild/base/testing.py | 2 +- easybuild/framework/__init__.py | 2 +- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/__init__.py | 2 +- easybuild/framework/easyconfig/constants.py | 2 +- easybuild/framework/easyconfig/default.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/format/__init__.py | 2 +- easybuild/framework/easyconfig/format/convert.py | 2 +- easybuild/framework/easyconfig/format/format.py | 2 +- easybuild/framework/easyconfig/format/one.py | 2 +- easybuild/framework/easyconfig/format/pyheaderconfigobj.py | 2 +- easybuild/framework/easyconfig/format/two.py | 2 +- easybuild/framework/easyconfig/format/version.py | 2 +- easybuild/framework/easyconfig/format/yeb.py | 2 +- easybuild/framework/easyconfig/licenses.py | 2 +- easybuild/framework/easyconfig/parser.py | 2 +- easybuild/framework/easyconfig/style.py | 2 +- easybuild/framework/easyconfig/templates.py | 2 +- easybuild/framework/easyconfig/tools.py | 2 +- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/framework/easyconfig/types.py | 2 +- easybuild/framework/easystack.py | 2 +- easybuild/framework/extension.py | 2 +- easybuild/framework/extensioneasyblock.py | 2 +- easybuild/main.py | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- easybuild/scripts/clean_gists.py | 2 +- easybuild/scripts/fix_docs.py | 2 +- easybuild/scripts/mk_tmpl_easyblock_for.py | 2 +- easybuild/scripts/rpath_args.py | 2 +- easybuild/scripts/rpath_wrapper_template.sh.in | 2 +- easybuild/toolchains/__init__.py | 2 +- easybuild/toolchains/cgmpich.py | 2 +- easybuild/toolchains/cgmpolf.py | 2 +- easybuild/toolchains/cgmvapich2.py | 2 +- easybuild/toolchains/cgmvolf.py | 2 +- easybuild/toolchains/cgompi.py | 2 +- easybuild/toolchains/cgoolf.py | 2 +- easybuild/toolchains/clanggcc.py | 2 +- easybuild/toolchains/compiler/__init__.py | 2 +- easybuild/toolchains/compiler/clang.py | 2 +- easybuild/toolchains/compiler/craype.py | 2 +- easybuild/toolchains/compiler/cuda.py | 2 +- easybuild/toolchains/compiler/dummycompiler.py | 2 +- easybuild/toolchains/compiler/fujitsu.py | 2 +- easybuild/toolchains/compiler/gcc.py | 2 +- easybuild/toolchains/compiler/intel_compilers.py | 2 +- easybuild/toolchains/compiler/inteliccifort.py | 2 +- easybuild/toolchains/compiler/nvhpc.py | 2 +- easybuild/toolchains/compiler/pgi.py | 2 +- easybuild/toolchains/compiler/systemcompiler.py | 2 +- easybuild/toolchains/craycce.py | 2 +- easybuild/toolchains/craygnu.py | 2 +- easybuild/toolchains/crayintel.py | 2 +- easybuild/toolchains/craypgi.py | 2 +- easybuild/toolchains/dummy.py | 2 +- easybuild/toolchains/fcc.py | 2 +- easybuild/toolchains/ffmpi.py | 2 +- easybuild/toolchains/fft/__init__.py | 2 +- easybuild/toolchains/fft/fftw.py | 2 +- easybuild/toolchains/fft/fujitsufftw.py | 2 +- easybuild/toolchains/fft/intelfftw.py | 2 +- easybuild/toolchains/foss.py | 2 +- easybuild/toolchains/fosscuda.py | 2 +- easybuild/toolchains/fujitsu.py | 2 +- easybuild/toolchains/gcc.py | 2 +- easybuild/toolchains/gcccore.py | 2 +- easybuild/toolchains/gcccuda.py | 2 +- easybuild/toolchains/gfbf.py | 2 +- easybuild/toolchains/gimkl.py | 2 +- easybuild/toolchains/gimpi.py | 2 +- easybuild/toolchains/gimpic.py | 2 +- easybuild/toolchains/giolf.py | 2 +- easybuild/toolchains/giolfc.py | 2 +- easybuild/toolchains/gmacml.py | 2 +- easybuild/toolchains/gmkl.py | 2 +- easybuild/toolchains/gmklc.py | 2 +- easybuild/toolchains/gmpich.py | 2 +- easybuild/toolchains/gmpich2.py | 2 +- easybuild/toolchains/gmpit.py | 2 +- easybuild/toolchains/gmpolf.py | 2 +- easybuild/toolchains/gmvapich2.py | 2 +- easybuild/toolchains/gmvolf.py | 2 +- easybuild/toolchains/gnu.py | 2 +- easybuild/toolchains/goalf.py | 2 +- easybuild/toolchains/gobff.py | 2 +- easybuild/toolchains/goblf.py | 2 +- easybuild/toolchains/gofbf.py | 2 +- easybuild/toolchains/golf.py | 2 +- easybuild/toolchains/golfc.py | 2 +- easybuild/toolchains/gomkl.py | 2 +- easybuild/toolchains/gomklc.py | 2 +- easybuild/toolchains/gompi.py | 2 +- easybuild/toolchains/gompic.py | 2 +- easybuild/toolchains/goolf.py | 2 +- easybuild/toolchains/goolfc.py | 2 +- easybuild/toolchains/gpsmpi.py | 2 +- easybuild/toolchains/gpsolf.py | 2 +- easybuild/toolchains/gqacml.py | 2 +- easybuild/toolchains/gsmpi.py | 2 +- easybuild/toolchains/gsolf.py | 2 +- easybuild/toolchains/iccifort.py | 2 +- easybuild/toolchains/iccifortcuda.py | 2 +- easybuild/toolchains/ictce.py | 2 +- easybuild/toolchains/ifbf.py | 2 +- easybuild/toolchains/iibff.py | 2 +- easybuild/toolchains/iimkl.py | 2 +- easybuild/toolchains/iimklc.py | 2 +- easybuild/toolchains/iimpi.py | 2 +- easybuild/toolchains/iimpic.py | 2 +- easybuild/toolchains/iiqmpi.py | 2 +- easybuild/toolchains/impich.py | 2 +- easybuild/toolchains/impmkl.py | 2 +- easybuild/toolchains/intel-para.py | 2 +- easybuild/toolchains/intel.py | 2 +- easybuild/toolchains/intel_compilers.py | 2 +- easybuild/toolchains/intelcuda.py | 2 +- easybuild/toolchains/iofbf.py | 2 +- easybuild/toolchains/iomkl.py | 2 +- easybuild/toolchains/iomklc.py | 2 +- easybuild/toolchains/iompi.py | 2 +- easybuild/toolchains/iompic.py | 2 +- easybuild/toolchains/ipsmpi.py | 2 +- easybuild/toolchains/iqacml.py | 2 +- easybuild/toolchains/ismkl.py | 2 +- easybuild/toolchains/linalg/__init__.py | 2 +- easybuild/toolchains/linalg/acml.py | 2 +- easybuild/toolchains/linalg/atlas.py | 2 +- easybuild/toolchains/linalg/blacs.py | 2 +- easybuild/toolchains/linalg/blis.py | 2 +- easybuild/toolchains/linalg/flame.py | 2 +- easybuild/toolchains/linalg/flexiblas.py | 2 +- easybuild/toolchains/linalg/fujitsussl.py | 2 +- easybuild/toolchains/linalg/gotoblas.py | 2 +- easybuild/toolchains/linalg/intelmkl.py | 2 +- easybuild/toolchains/linalg/lapack.py | 2 +- easybuild/toolchains/linalg/libsci.py | 2 +- easybuild/toolchains/linalg/openblas.py | 2 +- easybuild/toolchains/linalg/scalapack.py | 2 +- easybuild/toolchains/mpi/__init__.py | 2 +- easybuild/toolchains/mpi/craympich.py | 2 +- easybuild/toolchains/mpi/fujitsumpi.py | 2 +- easybuild/toolchains/mpi/intelmpi.py | 2 +- easybuild/toolchains/mpi/mpich.py | 2 +- easybuild/toolchains/mpi/mpich2.py | 2 +- easybuild/toolchains/mpi/mpitrampoline.py | 2 +- easybuild/toolchains/mpi/mvapich2.py | 2 +- easybuild/toolchains/mpi/openmpi.py | 2 +- easybuild/toolchains/mpi/psmpi.py | 2 +- easybuild/toolchains/mpi/qlogicmpi.py | 2 +- easybuild/toolchains/mpi/spectrummpi.py | 2 +- easybuild/toolchains/nvhpc.py | 2 +- easybuild/toolchains/nvofbf.py | 2 +- easybuild/toolchains/nvompi.py | 2 +- easybuild/toolchains/nvompic.py | 4 ++-- easybuild/toolchains/nvpsmpi.py | 4 ++-- easybuild/toolchains/nvpsmpic.py | 4 ++-- easybuild/toolchains/pgi.py | 2 +- easybuild/toolchains/pmkl.py | 2 +- easybuild/toolchains/pomkl.py | 2 +- easybuild/toolchains/pompi.py | 2 +- easybuild/toolchains/system.py | 2 +- easybuild/tools/__init__.py | 2 +- easybuild/tools/asyncprocess.py | 2 +- easybuild/tools/build_details.py | 2 +- easybuild/tools/build_log.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/containers/__init__.py | 2 +- easybuild/tools/containers/apptainer.py | 2 +- easybuild/tools/containers/base.py | 2 +- easybuild/tools/containers/common.py | 2 +- easybuild/tools/containers/docker.py | 2 +- easybuild/tools/containers/singularity.py | 2 +- easybuild/tools/containers/utils.py | 2 +- easybuild/tools/convert.py | 2 +- easybuild/tools/docs.py | 2 +- easybuild/tools/environment.py | 2 +- easybuild/tools/filetools.py | 2 +- easybuild/tools/github.py | 2 +- easybuild/tools/hooks.py | 2 +- easybuild/tools/include.py | 2 +- easybuild/tools/jenkins.py | 2 +- easybuild/tools/job/backend.py | 2 +- easybuild/tools/job/gc3pie.py | 2 +- easybuild/tools/job/pbs_python.py | 2 +- easybuild/tools/job/slurm.py | 2 +- easybuild/tools/module_generator.py | 2 +- easybuild/tools/module_naming_scheme/__init__.py | 2 +- easybuild/tools/module_naming_scheme/categorized_mns.py | 2 +- easybuild/tools/module_naming_scheme/easybuild_mns.py | 2 +- easybuild/tools/module_naming_scheme/hierarchical_mns.py | 2 +- .../tools/module_naming_scheme/migrate_from_eb_to_hmns.py | 2 +- easybuild/tools/module_naming_scheme/mns.py | 2 +- easybuild/tools/module_naming_scheme/toolchain.py | 2 +- easybuild/tools/module_naming_scheme/utilities.py | 2 +- easybuild/tools/modules.py | 2 +- easybuild/tools/multidiff.py | 2 +- easybuild/tools/options.py | 2 +- easybuild/tools/output.py | 2 +- .../package_naming_scheme/easybuild_deb_friendly_pns.py | 2 +- .../tools/package/package_naming_scheme/easybuild_pns.py | 2 +- easybuild/tools/package/package_naming_scheme/pns.py | 2 +- easybuild/tools/package/utilities.py | 2 +- easybuild/tools/parallelbuild.py | 2 +- easybuild/tools/py2vs3/__init__.py | 2 +- easybuild/tools/py2vs3/py2.py | 2 +- easybuild/tools/py2vs3/py3.py | 2 +- easybuild/tools/repository/filerepo.py | 2 +- easybuild/tools/repository/gitrepo.py | 2 +- easybuild/tools/repository/hgrepo.py | 2 +- easybuild/tools/repository/repository.py | 2 +- easybuild/tools/repository/svnrepo.py | 2 +- easybuild/tools/robot.py | 2 +- easybuild/tools/run.py | 2 +- easybuild/tools/systemtools.py | 2 +- easybuild/tools/testing.py | 2 +- easybuild/tools/toolchain/__init__.py | 2 +- easybuild/tools/toolchain/compiler.py | 2 +- easybuild/tools/toolchain/constants.py | 2 +- easybuild/tools/toolchain/fft.py | 2 +- easybuild/tools/toolchain/linalg.py | 2 +- easybuild/tools/toolchain/mpi.py | 2 +- easybuild/tools/toolchain/options.py | 2 +- easybuild/tools/toolchain/toolchain.py | 2 +- easybuild/tools/toolchain/toolchainvariables.py | 2 +- easybuild/tools/toolchain/utilities.py | 2 +- easybuild/tools/toolchain/variables.py | 2 +- easybuild/tools/utilities.py | 2 +- easybuild/tools/variables.py | 2 +- easybuild/tools/version.py | 2 +- eb | 2 +- setup.py | 2 +- test/__init__.py | 2 +- test/framework/__init__.py | 2 +- test/framework/asyncprocess.py | 2 +- test/framework/build_log.py | 2 +- test/framework/config.py | 2 +- test/framework/containers.py | 2 +- test/framework/docs.py | 2 +- test/framework/easyblock.py | 2 +- test/framework/easyconfig.py | 2 +- test/framework/easyconfigformat.py | 2 +- test/framework/easyconfigparser.py | 2 +- test/framework/easyconfigversion.py | 2 +- test/framework/easystack.py | 2 +- test/framework/ebconfigobj.py | 2 +- test/framework/environment.py | 2 +- test/framework/filetools.py | 2 +- test/framework/format_convert.py | 2 +- test/framework/general.py | 2 +- test/framework/github.py | 2 +- test/framework/hooks.py | 2 +- test/framework/include.py | 2 +- test/framework/lib.py | 2 +- test/framework/license.py | 2 +- test/framework/module_generator.py | 2 +- test/framework/modules.py | 2 +- test/framework/modulestool.py | 2 +- test/framework/options.py | 2 +- test/framework/output.py | 2 +- test/framework/package.py | 2 +- test/framework/parallelbuild.py | 2 +- test/framework/repository.py | 2 +- test/framework/robot.py | 2 +- test/framework/run.py | 2 +- .../framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/fftw.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/f/foofoo.py | 2 +- test/framework/sandbox/easybuild/easyblocks/g/gcc.py | 2 +- test/framework/sandbox/easybuild/easyblocks/generic/bar.py | 2 +- .../sandbox/easybuild/easyblocks/generic/configuremake.py | 2 +- .../sandbox/easybuild/easyblocks/generic/dummyextension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/generic/makecp.py | 2 +- .../sandbox/easybuild/easyblocks/generic/modulerc.py | 2 +- .../sandbox/easybuild/easyblocks/generic/pythonbundle.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toolchain.py | 2 +- .../sandbox/easybuild/easyblocks/generic/toy_extension.py | 2 +- test/framework/sandbox/easybuild/easyblocks/h/hpl.py | 2 +- test/framework/sandbox/easybuild/easyblocks/l/libtoy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/o/openblas.py | 2 +- test/framework/sandbox/easybuild/easyblocks/o/openmpi.py | 2 +- test/framework/sandbox/easybuild/easyblocks/s/scalapack.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py | 2 +- test/framework/sandbox/easybuild/easyblocks/t/toytoy.py | 2 +- test/framework/sandbox/easybuild/tools/__init__.py | 2 +- .../sandbox/easybuild/tools/module_naming_scheme/__init__.py | 2 +- .../tools/module_naming_scheme/broken_module_naming_scheme.py | 2 +- .../tools/module_naming_scheme/test_module_naming_scheme.py | 2 +- .../module_naming_scheme/test_module_naming_scheme_more.py | 2 +- test/framework/style.py | 2 +- test/framework/suite.py | 2 +- test/framework/systemtools.py | 2 +- test/framework/toolchain.py | 2 +- test/framework/toolchainvariables.py | 2 +- test/framework/toy_build.py | 2 +- test/framework/tweak.py | 2 +- test/framework/type_checking.py | 2 +- test/framework/utilities.py | 2 +- test/framework/utilities_test.py | 2 +- test/framework/variables.py | 2 +- test/framework/yeb.py | 2 +- 309 files changed, 312 insertions(+), 312 deletions(-) diff --git a/easybuild/__init__.py b/easybuild/__init__.py index 4763af8bc4..3637c9f395 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/exceptions.py b/easybuild/base/exceptions.py index 3bde7cc6a1..eb004ad217 100644 --- a/easybuild/base/exceptions.py +++ b/easybuild/base/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index 93582b0cf1..4efe6ee615 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index c5af9220d1..cfd1a37e1f 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1,5 +1,5 @@ # -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 01b64d84ef..4b3df7e563 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -1,5 +1,5 @@ # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/__init__.py b/easybuild/framework/__init__.py index 019e507d79..f03298abca 100644 --- a/easybuild/framework/__init__.py +++ b/easybuild/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 063b2c01cc..d2727d61ee 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/__init__.py b/easybuild/framework/easyconfig/__init__.py index 6d91c028a9..b19017e023 100644 --- a/easybuild/framework/easyconfig/__init__.py +++ b/easybuild/framework/easyconfig/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/constants.py b/easybuild/framework/easyconfig/constants.py index 2e62580db4..4fe8c9d7b5 100644 --- a/easybuild/framework/easyconfig/constants.py +++ b/easybuild/framework/easyconfig/constants.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index dae27aca65..172995246e 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ff4ed3562c..6e5f6aae35 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/__init__.py b/easybuild/framework/easyconfig/format/__init__.py index b2c084e273..c2a81766ad 100644 --- a/easybuild/framework/easyconfig/format/__init__.py +++ b/easybuild/framework/easyconfig/format/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/convert.py b/easybuild/framework/easyconfig/format/convert.py index 726acd3e6c..82613d42e0 100644 --- a/easybuild/framework/easyconfig/format/convert.py +++ b/easybuild/framework/easyconfig/format/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index d503b6703a..3aeae419a2 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 680e4ee77b..868e218925 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 37e14b529e..f5889f5e2c 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/two.py b/easybuild/framework/easyconfig/format/two.py index cec3df3636..652b1c9b80 100644 --- a/easybuild/framework/easyconfig/format/two.py +++ b/easybuild/framework/easyconfig/format/two.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/version.py b/easybuild/framework/easyconfig/format/version.py index fe3b1a2316..54d05de9f9 100644 --- a/easybuild/framework/easyconfig/format/version.py +++ b/easybuild/framework/easyconfig/format/version.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index acfab3fb22..675d0f0a86 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/licenses.py b/easybuild/framework/easyconfig/licenses.py index e8eb1ee3bc..e6286d54e3 100644 --- a/easybuild/framework/easyconfig/licenses.py +++ b/easybuild/framework/easyconfig/licenses.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index fa01d1b297..d6521b17dc 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/style.py b/easybuild/framework/easyconfig/style.py index 0015b0c9b9..bad6ab5bda 100644 --- a/easybuild/framework/easyconfig/style.py +++ b/easybuild/framework/easyconfig/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index aed8db6af4..80d38e8bad 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -1,5 +1,5 @@ # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 73393040c6..7d461c1461 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 7aaa69b69a..fb01f3a83e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 622479f480..42be1b99e4 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 9296c18827..4a0ec15cdd 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -1,4 +1,4 @@ -# Copyright 2020-2023 Ghent University +# Copyright 2020-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index ed90f15e6d..56f242ffce 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 6ad116b20b..69c824fe7e 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). diff --git a/easybuild/main.py b/easybuild/main.py index 85501f57f0..41eefb4461 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index a74f4fa55e..ef8c281162 100755 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index cb2d6fed04..ec1213e4b3 100755 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2014 Ward Poelmans +# Copyright 2014-2024 Ward Poelmans # # https://github.com/easybuilders/easybuild # diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py index 388fb360d9..a14f18caf8 100755 --- a/easybuild/scripts/fix_docs.py +++ b/easybuild/scripts/fix_docs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index c5c7c0110a..f6958572fa 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 1bc87a1679..b477aa63d8 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 55c3388c5b..44a5aac43a 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/__init__.py b/easybuild/toolchains/__init__.py index 309b1808cf..2f468f2812 100644 --- a/easybuild/toolchains/__init__.py +++ b/easybuild/toolchains/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/cgmpich.py b/easybuild/toolchains/cgmpich.py index ea04e20e75..9e25c6ec57 100644 --- a/easybuild/toolchains/cgmpich.py +++ b/easybuild/toolchains/cgmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmpolf.py b/easybuild/toolchains/cgmpolf.py index b335781374..5629ee5be3 100644 --- a/easybuild/toolchains/cgmpolf.py +++ b/easybuild/toolchains/cgmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvapich2.py b/easybuild/toolchains/cgmvapich2.py index e347cab483..76e306b12d 100644 --- a/easybuild/toolchains/cgmvapich2.py +++ b/easybuild/toolchains/cgmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgmvolf.py b/easybuild/toolchains/cgmvolf.py index 1b9e122fcd..e68aa90fe1 100644 --- a/easybuild/toolchains/cgmvolf.py +++ b/easybuild/toolchains/cgmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgompi.py b/easybuild/toolchains/cgompi.py index 559f91f4ad..9b9267bf78 100644 --- a/easybuild/toolchains/cgompi.py +++ b/easybuild/toolchains/cgompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/cgoolf.py b/easybuild/toolchains/cgoolf.py index c26e94b724..8e2ea8503e 100644 --- a/easybuild/toolchains/cgoolf.py +++ b/easybuild/toolchains/cgoolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/clanggcc.py b/easybuild/toolchains/clanggcc.py index 8537354ebe..c1e09e68ef 100644 --- a/easybuild/toolchains/clanggcc.py +++ b/easybuild/toolchains/clanggcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/__init__.py b/easybuild/toolchains/compiler/__init__.py index 5933777f43..9c1ba469c5 100644 --- a/easybuild/toolchains/compiler/__init__.py +++ b/easybuild/toolchains/compiler/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 8c93053932..8b7b16e8e3 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/craype.py b/easybuild/toolchains/compiler/craype.py index a0bec9f624..73b6a103e2 100644 --- a/easybuild/toolchains/compiler/craype.py +++ b/easybuild/toolchains/compiler/craype.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index c87f1b21d1..3fd3d705f1 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/dummycompiler.py b/easybuild/toolchains/compiler/dummycompiler.py index 767dcb66c5..d74c2e873e 100644 --- a/easybuild/toolchains/compiler/dummycompiler.py +++ b/easybuild/toolchains/compiler/dummycompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/fujitsu.py b/easybuild/toolchains/compiler/fujitsu.py index e7a861c1b7..437ec34048 100644 --- a/easybuild/toolchains/compiler/fujitsu.py +++ b/easybuild/toolchains/compiler/fujitsu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index fd20846592..548ac41187 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index ae97dfa87d..1a21b21943 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/inteliccifort.py b/easybuild/toolchains/compiler/inteliccifort.py index c498f544d7..01015a0d83 100644 --- a/easybuild/toolchains/compiler/inteliccifort.py +++ b/easybuild/toolchains/compiler/inteliccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/compiler/nvhpc.py b/easybuild/toolchains/compiler/nvhpc.py index 549c0e9a70..2011137716 100644 --- a/easybuild/toolchains/compiler/nvhpc.py +++ b/easybuild/toolchains/compiler/nvhpc.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/pgi.py b/easybuild/toolchains/compiler/pgi.py index 77f04fe813..0f55614926 100644 --- a/easybuild/toolchains/compiler/pgi.py +++ b/easybuild/toolchains/compiler/pgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/compiler/systemcompiler.py b/easybuild/toolchains/compiler/systemcompiler.py index 505f002d57..d1fd06d4d9 100644 --- a/easybuild/toolchains/compiler/systemcompiler.py +++ b/easybuild/toolchains/compiler/systemcompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craycce.py b/easybuild/toolchains/craycce.py index cb9a4ff80e..13b92e55d2 100644 --- a/easybuild/toolchains/craycce.py +++ b/easybuild/toolchains/craycce.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craygnu.py b/easybuild/toolchains/craygnu.py index 4e42d32f94..8d198362bd 100644 --- a/easybuild/toolchains/craygnu.py +++ b/easybuild/toolchains/craygnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/crayintel.py b/easybuild/toolchains/crayintel.py index 76cde0a8bc..5997eb63d6 100644 --- a/easybuild/toolchains/crayintel.py +++ b/easybuild/toolchains/crayintel.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/craypgi.py b/easybuild/toolchains/craypgi.py index 67e2bcec78..49004775a4 100644 --- a/easybuild/toolchains/craypgi.py +++ b/easybuild/toolchains/craypgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/dummy.py b/easybuild/toolchains/dummy.py index cb2a988bd0..f29f6736ae 100644 --- a/easybuild/toolchains/dummy.py +++ b/easybuild/toolchains/dummy.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fcc.py b/easybuild/toolchains/fcc.py index 905835e3cf..bc99335c48 100644 --- a/easybuild/toolchains/fcc.py +++ b/easybuild/toolchains/fcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ffmpi.py b/easybuild/toolchains/ffmpi.py index 34a56d229e..71a49f7913 100644 --- a/easybuild/toolchains/ffmpi.py +++ b/easybuild/toolchains/ffmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/__init__.py b/easybuild/toolchains/fft/__init__.py index 9915268575..afcf0211dd 100644 --- a/easybuild/toolchains/fft/__init__.py +++ b/easybuild/toolchains/fft/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fftw.py b/easybuild/toolchains/fft/fftw.py index 63d39cb658..9cb4c4e851 100644 --- a/easybuild/toolchains/fft/fftw.py +++ b/easybuild/toolchains/fft/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/fujitsufftw.py b/easybuild/toolchains/fft/fujitsufftw.py index 1e2ed28ec6..4de1769528 100644 --- a/easybuild/toolchains/fft/fujitsufftw.py +++ b/easybuild/toolchains/fft/fujitsufftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fft/intelfftw.py b/easybuild/toolchains/fft/intelfftw.py index ad8f08ffd8..7882278a91 100644 --- a/easybuild/toolchains/fft/intelfftw.py +++ b/easybuild/toolchains/fft/intelfftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/foss.py b/easybuild/toolchains/foss.py index d5e57956a0..031daa4c9c 100644 --- a/easybuild/toolchains/foss.py +++ b/easybuild/toolchains/foss.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fosscuda.py b/easybuild/toolchains/fosscuda.py index a35c570781..9465515d36 100644 --- a/easybuild/toolchains/fosscuda.py +++ b/easybuild/toolchains/fosscuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/fujitsu.py b/easybuild/toolchains/fujitsu.py index ed858a84c7..707255e03c 100644 --- a/easybuild/toolchains/fujitsu.py +++ b/easybuild/toolchains/fujitsu.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcc.py b/easybuild/toolchains/gcc.py index e359b8aa01..71091e795c 100644 --- a/easybuild/toolchains/gcc.py +++ b/easybuild/toolchains/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccore.py b/easybuild/toolchains/gcccore.py index 87eaf4d1b6..ce4703e89c 100644 --- a/easybuild/toolchains/gcccore.py +++ b/easybuild/toolchains/gcccore.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gcccuda.py b/easybuild/toolchains/gcccuda.py index fcf0981bac..47ab250824 100644 --- a/easybuild/toolchains/gcccuda.py +++ b/easybuild/toolchains/gcccuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gfbf.py b/easybuild/toolchains/gfbf.py index 9a7f279e6d..d41f5b7d06 100644 --- a/easybuild/toolchains/gfbf.py +++ b/easybuild/toolchains/gfbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimkl.py b/easybuild/toolchains/gimkl.py index 8505cf5b87..1d7b9704f6 100644 --- a/easybuild/toolchains/gimkl.py +++ b/easybuild/toolchains/gimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpi.py b/easybuild/toolchains/gimpi.py index 6ac9788749..76e6bd638a 100644 --- a/easybuild/toolchains/gimpi.py +++ b/easybuild/toolchains/gimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gimpic.py b/easybuild/toolchains/gimpic.py index 01a28c78ee..dde1d6c8f8 100644 --- a/easybuild/toolchains/gimpic.py +++ b/easybuild/toolchains/gimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolf.py b/easybuild/toolchains/giolf.py index 64b99f2a5c..121c81121b 100644 --- a/easybuild/toolchains/giolf.py +++ b/easybuild/toolchains/giolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/giolfc.py b/easybuild/toolchains/giolfc.py index 3ca3111542..df798626aa 100644 --- a/easybuild/toolchains/giolfc.py +++ b/easybuild/toolchains/giolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmacml.py b/easybuild/toolchains/gmacml.py index 5561f85286..ee95c3f7f3 100644 --- a/easybuild/toolchains/gmacml.py +++ b/easybuild/toolchains/gmacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmkl.py b/easybuild/toolchains/gmkl.py index a58a05c488..67e33e1e62 100644 --- a/easybuild/toolchains/gmkl.py +++ b/easybuild/toolchains/gmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmklc.py b/easybuild/toolchains/gmklc.py index b7ecd19115..bead896d5e 100644 --- a/easybuild/toolchains/gmklc.py +++ b/easybuild/toolchains/gmklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich.py b/easybuild/toolchains/gmpich.py index 941421e50e..66d2b29123 100644 --- a/easybuild/toolchains/gmpich.py +++ b/easybuild/toolchains/gmpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpich2.py b/easybuild/toolchains/gmpich2.py index bad4e0bbaf..f734d3cb56 100644 --- a/easybuild/toolchains/gmpich2.py +++ b/easybuild/toolchains/gmpich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpit.py b/easybuild/toolchains/gmpit.py index 395940a639..a39baf934d 100644 --- a/easybuild/toolchains/gmpit.py +++ b/easybuild/toolchains/gmpit.py @@ -1,5 +1,5 @@ ## -# Copyright 2022-2023 Ghent University +# Copyright 2022-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmpolf.py b/easybuild/toolchains/gmpolf.py index 2bca2b44dd..c5a8be97d9 100644 --- a/easybuild/toolchains/gmpolf.py +++ b/easybuild/toolchains/gmpolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gmvapich2.py b/easybuild/toolchains/gmvapich2.py index 46ebb4d489..26aa6ad73e 100644 --- a/easybuild/toolchains/gmvapich2.py +++ b/easybuild/toolchains/gmvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gmvolf.py b/easybuild/toolchains/gmvolf.py index 50c943cde6..613d8a6ed3 100644 --- a/easybuild/toolchains/gmvolf.py +++ b/easybuild/toolchains/gmvolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gnu.py b/easybuild/toolchains/gnu.py index a16eed38bd..84b06af535 100644 --- a/easybuild/toolchains/gnu.py +++ b/easybuild/toolchains/gnu.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goalf.py b/easybuild/toolchains/goalf.py index d833e3c1c5..db1280907b 100644 --- a/easybuild/toolchains/goalf.py +++ b/easybuild/toolchains/goalf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py index 2cbb366dfe..9df26ff706 100644 --- a/easybuild/toolchains/gobff.py +++ b/easybuild/toolchains/gobff.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goblf.py b/easybuild/toolchains/goblf.py index 618d813149..63e7c0c0a4 100644 --- a/easybuild/toolchains/goblf.py +++ b/easybuild/toolchains/goblf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gofbf.py b/easybuild/toolchains/gofbf.py index 71269a57df..3128c87091 100644 --- a/easybuild/toolchains/gofbf.py +++ b/easybuild/toolchains/gofbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/golf.py b/easybuild/toolchains/golf.py index c318ec9fee..6d046413a8 100644 --- a/easybuild/toolchains/golf.py +++ b/easybuild/toolchains/golf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/golfc.py b/easybuild/toolchains/golfc.py index bdce5ddcec..3b99dce5c0 100644 --- a/easybuild/toolchains/golfc.py +++ b/easybuild/toolchains/golfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomkl.py b/easybuild/toolchains/gomkl.py index 663bc32daa..ba93882a1b 100644 --- a/easybuild/toolchains/gomkl.py +++ b/easybuild/toolchains/gomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gomklc.py b/easybuild/toolchains/gomklc.py index fa19da4b50..95620d8360 100644 --- a/easybuild/toolchains/gomklc.py +++ b/easybuild/toolchains/gomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompi.py b/easybuild/toolchains/gompi.py index 35db8a8330..7af87123c9 100644 --- a/easybuild/toolchains/gompi.py +++ b/easybuild/toolchains/gompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gompic.py b/easybuild/toolchains/gompic.py index a9f08dcf2a..0beb2b4d10 100644 --- a/easybuild/toolchains/gompic.py +++ b/easybuild/toolchains/gompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolf.py b/easybuild/toolchains/goolf.py index 6577cb31c4..099f7bc11a 100644 --- a/easybuild/toolchains/goolf.py +++ b/easybuild/toolchains/goolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/goolfc.py b/easybuild/toolchains/goolfc.py index 26cd18ec9b..76cf4db890 100644 --- a/easybuild/toolchains/goolfc.py +++ b/easybuild/toolchains/goolfc.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsmpi.py b/easybuild/toolchains/gpsmpi.py index c6a7fd04b8..529d17dc5e 100644 --- a/easybuild/toolchains/gpsmpi.py +++ b/easybuild/toolchains/gpsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gpsolf.py b/easybuild/toolchains/gpsolf.py index 1414178019..e87f3a5de7 100644 --- a/easybuild/toolchains/gpsolf.py +++ b/easybuild/toolchains/gpsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/gqacml.py b/easybuild/toolchains/gqacml.py index 5533bd1f3a..7456b4011e 100644 --- a/easybuild/toolchains/gqacml.py +++ b/easybuild/toolchains/gqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsmpi.py b/easybuild/toolchains/gsmpi.py index 865dbc813b..68f1a6d0f0 100644 --- a/easybuild/toolchains/gsmpi.py +++ b/easybuild/toolchains/gsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/gsolf.py b/easybuild/toolchains/gsolf.py index 5d30493bb1..cb917ca7d3 100644 --- a/easybuild/toolchains/gsolf.py +++ b/easybuild/toolchains/gsolf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifort.py b/easybuild/toolchains/iccifort.py index d1b6804a93..38ec224a5b 100644 --- a/easybuild/toolchains/iccifort.py +++ b/easybuild/toolchains/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iccifortcuda.py b/easybuild/toolchains/iccifortcuda.py index 5f024dc8ce..149cfc8e39 100644 --- a/easybuild/toolchains/iccifortcuda.py +++ b/easybuild/toolchains/iccifortcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ictce.py b/easybuild/toolchains/ictce.py index 265f249306..2055346b37 100644 --- a/easybuild/toolchains/ictce.py +++ b/easybuild/toolchains/ictce.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py index 3507e1eab9..9f689279cb 100644 --- a/easybuild/toolchains/ifbf.py +++ b/easybuild/toolchains/ifbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iibff.py b/easybuild/toolchains/iibff.py index 9fe2b2b0c8..07a6b1e567 100644 --- a/easybuild/toolchains/iibff.py +++ b/easybuild/toolchains/iibff.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimkl.py b/easybuild/toolchains/iimkl.py index 9d3d6a97f3..605445cc5c 100644 --- a/easybuild/toolchains/iimkl.py +++ b/easybuild/toolchains/iimkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimklc.py b/easybuild/toolchains/iimklc.py index e9508c95ad..89325c2e67 100644 --- a/easybuild/toolchains/iimklc.py +++ b/easybuild/toolchains/iimklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpi.py b/easybuild/toolchains/iimpi.py index 00d9745a42..9337365ec6 100644 --- a/easybuild/toolchains/iimpi.py +++ b/easybuild/toolchains/iimpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iimpic.py b/easybuild/toolchains/iimpic.py index 53ca437150..45a8df3804 100644 --- a/easybuild/toolchains/iimpic.py +++ b/easybuild/toolchains/iimpic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iiqmpi.py b/easybuild/toolchains/iiqmpi.py index aee09bb658..5ff1466ff3 100644 --- a/easybuild/toolchains/iiqmpi.py +++ b/easybuild/toolchains/iiqmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impich.py b/easybuild/toolchains/impich.py index 2b7fba07f9..7feda33805 100644 --- a/easybuild/toolchains/impich.py +++ b/easybuild/toolchains/impich.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/impmkl.py b/easybuild/toolchains/impmkl.py index 1653b31d5c..952bff34ce 100644 --- a/easybuild/toolchains/impmkl.py +++ b/easybuild/toolchains/impmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel-para.py b/easybuild/toolchains/intel-para.py index a9d4ca0313..ddfee3aaae 100644 --- a/easybuild/toolchains/intel-para.py +++ b/easybuild/toolchains/intel-para.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel.py b/easybuild/toolchains/intel.py index 7dd625588f..659b05faf2 100644 --- a/easybuild/toolchains/intel.py +++ b/easybuild/toolchains/intel.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intel_compilers.py b/easybuild/toolchains/intel_compilers.py index ab5de03386..27fdfbebfd 100644 --- a/easybuild/toolchains/intel_compilers.py +++ b/easybuild/toolchains/intel_compilers.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/intelcuda.py b/easybuild/toolchains/intelcuda.py index 343715e43b..bdd59abaab 100644 --- a/easybuild/toolchains/intelcuda.py +++ b/easybuild/toolchains/intelcuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py index 3410ffaf9f..cd3313d600 100644 --- a/easybuild/toolchains/iofbf.py +++ b/easybuild/toolchains/iofbf.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomkl.py b/easybuild/toolchains/iomkl.py index cefe83c3b1..033277919c 100644 --- a/easybuild/toolchains/iomkl.py +++ b/easybuild/toolchains/iomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iomklc.py b/easybuild/toolchains/iomklc.py index 102fa7a311..f40cd78e16 100644 --- a/easybuild/toolchains/iomklc.py +++ b/easybuild/toolchains/iomklc.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompi.py b/easybuild/toolchains/iompi.py index 318db78481..b578c45892 100644 --- a/easybuild/toolchains/iompi.py +++ b/easybuild/toolchains/iompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iompic.py b/easybuild/toolchains/iompic.py index 1bcb2eac71..5b675f5621 100644 --- a/easybuild/toolchains/iompic.py +++ b/easybuild/toolchains/iompic.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ipsmpi.py b/easybuild/toolchains/ipsmpi.py index 1a381414c3..a8a772adcb 100644 --- a/easybuild/toolchains/ipsmpi.py +++ b/easybuild/toolchains/ipsmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/iqacml.py b/easybuild/toolchains/iqacml.py index fc63968583..e4c5476cd4 100644 --- a/easybuild/toolchains/iqacml.py +++ b/easybuild/toolchains/iqacml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/ismkl.py b/easybuild/toolchains/ismkl.py index 90ff100eeb..4ccdc87054 100644 --- a/easybuild/toolchains/ismkl.py +++ b/easybuild/toolchains/ismkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/__init__.py b/easybuild/toolchains/linalg/__init__.py index 152bae1e3d..90cdb068a8 100644 --- a/easybuild/toolchains/linalg/__init__.py +++ b/easybuild/toolchains/linalg/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/acml.py b/easybuild/toolchains/linalg/acml.py index b9e25b311d..268e3b0cc2 100644 --- a/easybuild/toolchains/linalg/acml.py +++ b/easybuild/toolchains/linalg/acml.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/atlas.py b/easybuild/toolchains/linalg/atlas.py index 4d53f5bd32..bc894bb108 100644 --- a/easybuild/toolchains/linalg/atlas.py +++ b/easybuild/toolchains/linalg/atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blacs.py b/easybuild/toolchains/linalg/blacs.py index a3963e9daa..201681c51f 100644 --- a/easybuild/toolchains/linalg/blacs.py +++ b/easybuild/toolchains/linalg/blacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index fa607d0527..d89f3b8d83 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 78ac8f1ca4..5f6f5cfe87 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index c266aff248..0e88c1a9b2 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/fujitsussl.py b/easybuild/toolchains/linalg/fujitsussl.py index ecaff56683..dde792455b 100644 --- a/easybuild/toolchains/linalg/fujitsussl.py +++ b/easybuild/toolchains/linalg/fujitsussl.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/gotoblas.py b/easybuild/toolchains/linalg/gotoblas.py index 17d98413a8..e8a99a651a 100644 --- a/easybuild/toolchains/linalg/gotoblas.py +++ b/easybuild/toolchains/linalg/gotoblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 201355be46..8a32d684d9 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/lapack.py b/easybuild/toolchains/linalg/lapack.py index e9821a659d..e26c9e2961 100644 --- a/easybuild/toolchains/linalg/lapack.py +++ b/easybuild/toolchains/linalg/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index 87c42116f5..07ea64ed82 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/openblas.py b/easybuild/toolchains/linalg/openblas.py index e28d4f7dd2..18118d1f01 100644 --- a/easybuild/toolchains/linalg/openblas.py +++ b/easybuild/toolchains/linalg/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/linalg/scalapack.py b/easybuild/toolchains/linalg/scalapack.py index ef4d24ef30..980e3bfdab 100644 --- a/easybuild/toolchains/linalg/scalapack.py +++ b/easybuild/toolchains/linalg/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/__init__.py b/easybuild/toolchains/mpi/__init__.py index aea95e9051..c5abd0f595 100644 --- a/easybuild/toolchains/mpi/__init__.py +++ b/easybuild/toolchains/mpi/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/craympich.py b/easybuild/toolchains/mpi/craympich.py index 4c62c45519..14fc8cfcd9 100644 --- a/easybuild/toolchains/mpi/craympich.py +++ b/easybuild/toolchains/mpi/craympich.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/fujitsumpi.py b/easybuild/toolchains/mpi/fujitsumpi.py index f81e1d66f0..769ce94507 100644 --- a/easybuild/toolchains/mpi/fujitsumpi.py +++ b/easybuild/toolchains/mpi/fujitsumpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index 6b9f86430a..8bc67fe788 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich.py b/easybuild/toolchains/mpi/mpich.py index b0c42e57d8..51c926173f 100644 --- a/easybuild/toolchains/mpi/mpich.py +++ b/easybuild/toolchains/mpi/mpich.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpich2.py b/easybuild/toolchains/mpi/mpich2.py index bd85d30532..b6f1f6f082 100644 --- a/easybuild/toolchains/mpi/mpich2.py +++ b/easybuild/toolchains/mpi/mpich2.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mpitrampoline.py b/easybuild/toolchains/mpi/mpitrampoline.py index 5af054661c..7ded1102e5 100644 --- a/easybuild/toolchains/mpi/mpitrampoline.py +++ b/easybuild/toolchains/mpi/mpitrampoline.py @@ -1,5 +1,5 @@ ## -# Copyright 2022-2023 Ghent University +# Copyright 2022-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/mvapich2.py b/easybuild/toolchains/mpi/mvapich2.py index afb2602f44..4f69c970f5 100644 --- a/easybuild/toolchains/mpi/mvapich2.py +++ b/easybuild/toolchains/mpi/mvapich2.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/openmpi.py b/easybuild/toolchains/mpi/openmpi.py index 630262f48b..93531f3355 100644 --- a/easybuild/toolchains/mpi/openmpi.py +++ b/easybuild/toolchains/mpi/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/psmpi.py b/easybuild/toolchains/mpi/psmpi.py index cfbf720a74..07e8c9af78 100644 --- a/easybuild/toolchains/mpi/psmpi.py +++ b/easybuild/toolchains/mpi/psmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/qlogicmpi.py b/easybuild/toolchains/mpi/qlogicmpi.py index 26c0663756..b7355cd3e1 100644 --- a/easybuild/toolchains/mpi/qlogicmpi.py +++ b/easybuild/toolchains/mpi/qlogicmpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/mpi/spectrummpi.py b/easybuild/toolchains/mpi/spectrummpi.py index e0d4b2f530..1fee1d51b7 100644 --- a/easybuild/toolchains/mpi/spectrummpi.py +++ b/easybuild/toolchains/mpi/spectrummpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvhpc.py b/easybuild/toolchains/nvhpc.py index aca69d386f..b4b871f84f 100644 --- a/easybuild/toolchains/nvhpc.py +++ b/easybuild/toolchains/nvhpc.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/nvofbf.py b/easybuild/toolchains/nvofbf.py index 41e89e5a32..62d3cb4d37 100644 --- a/easybuild/toolchains/nvofbf.py +++ b/easybuild/toolchains/nvofbf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py index 5f2e25f03d..3feedaae35 100644 --- a/easybuild/toolchains/nvompi.py +++ b/easybuild/toolchains/nvompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2021 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py index e4ac0d6106..b9c5abc298 100644 --- a/easybuild/toolchains/nvompic.py +++ b/easybuild/toolchains/nvompic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2023 Ghent University -# Copyright 2016-2023 Forschungszentrum Juelich +# Copyright 2016-2024 Ghent University +# Copyright 2016-2024 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py index 9e102c9d53..cbaee42bc0 100644 --- a/easybuild/toolchains/nvpsmpi.py +++ b/easybuild/toolchains/nvpsmpi.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2021 Ghent University -# Copyright 2016-2021 Forschungszentrum Juelich +# Copyright 2016-2024 Ghent University +# Copyright 2016-2024 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index 604630ff8e..0b916c30a0 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -1,6 +1,6 @@ ## -# Copyright 2016-2023 Ghent University -# Copyright 2016-2023 Forschungszentrum Juelich +# Copyright 2016-2024 Ghent University +# Copyright 2016-2024 Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pgi.py b/easybuild/toolchains/pgi.py index 2ee3dc3839..df2258bda4 100644 --- a/easybuild/toolchains/pgi.py +++ b/easybuild/toolchains/pgi.py @@ -1,5 +1,5 @@ ## -# Copyright 2015 Bart Oldeman +# Copyright 2015-2024 Bart Oldeman # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. diff --git a/easybuild/toolchains/pmkl.py b/easybuild/toolchains/pmkl.py index a4ad73d7cd..8b22d1d35f 100644 --- a/easybuild/toolchains/pmkl.py +++ b/easybuild/toolchains/pmkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pomkl.py b/easybuild/toolchains/pomkl.py index ea85e2d440..f28f445296 100644 --- a/easybuild/toolchains/pomkl.py +++ b/easybuild/toolchains/pomkl.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/pompi.py b/easybuild/toolchains/pompi.py index f8a9e00039..9de2e52050 100644 --- a/easybuild/toolchains/pompi.py +++ b/easybuild/toolchains/pompi.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/toolchains/system.py b/easybuild/toolchains/system.py index cfb46a93b2..8a8b17cfaa 100644 --- a/easybuild/toolchains/system.py +++ b/easybuild/toolchains/system.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/__init__.py b/easybuild/tools/__init__.py index dee4fc0d12..8d34fc25c5 100644 --- a/easybuild/tools/__init__.py +++ b/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index b3a7d330ff..0a3c133fc4 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -1,6 +1,6 @@ ## # Copyright 2005 Josiah Carlson -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # The Asynchronous Python Subprocess recipe was originally created by Josiah Carlson. # and released under the GPL v2 on March 14, 2012 diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index e99ddb4af9..487e6372a3 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -1,4 +1,4 @@ -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 215b8d38aa..0bbff702a1 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ee8eb5fc53..b00ff1f8b3 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/__init__.py b/easybuild/tools/containers/__init__.py index 76cb37219a..215ba61c0c 100644 --- a/easybuild/tools/containers/__init__.py +++ b/easybuild/tools/containers/__init__.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py index 67a6db5bc3..4cb252dac4 100644 --- a/easybuild/tools/containers/apptainer.py +++ b/easybuild/tools/containers/apptainer.py @@ -1,4 +1,4 @@ -# Copyright 2022-2023 Ghent University +# Copyright 2022-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/base.py b/easybuild/tools/containers/base.py index 68a49610d3..faf3b0cc7a 100644 --- a/easybuild/tools/containers/base.py +++ b/easybuild/tools/containers/base.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/common.py b/easybuild/tools/containers/common.py index bcbdce9a9b..6824c16864 100644 --- a/easybuild/tools/containers/common.py +++ b/easybuild/tools/containers/common.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index aa37a90873..9061bba762 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index dc9cbb39a1..809fdc35d1 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -1,4 +1,4 @@ -# Copyright 2017-2023 Ghent University +# Copyright 2017-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index 65be9dadf5..a575254335 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/convert.py b/easybuild/tools/convert.py index f085590988..134566812d 100644 --- a/easybuild/tools/convert.py +++ b/easybuild/tools/convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 13f2c11c27..d610bc6e87 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index d68755ff24..db6a488183 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index bc938f4210..18784e0cce 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 96ed4d6492..5ffc95c1fb 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index f454974edb..f018c40607 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2023 Ghent University +# Copyright 2017-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 3898facd4c..0171d74f10 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index 3c00d3961c..50ee563083 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index c92ead6a58..1629ff3504 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/gc3pie.py b/easybuild/tools/job/gc3pie.py index 005cff2e90..f6b57e0f2c 100644 --- a/easybuild/tools/job/gc3pie.py +++ b/easybuild/tools/job/gc3pie.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # Copyright 2015 S3IT, University of Zurich # # This file is part of EasyBuild, diff --git a/easybuild/tools/job/pbs_python.py b/easybuild/tools/job/pbs_python.py index a4a020988e..308067ef7d 100644 --- a/easybuild/tools/job/pbs_python.py +++ b/easybuild/tools/job/pbs_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py index 1f2dea776d..2666cbd6af 100644 --- a/easybuild/tools/job/slurm.py +++ b/easybuild/tools/job/slurm.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2023 Ghent University +# Copyright 2018-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 45582bbe5c..7f9344f5d8 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/__init__.py b/easybuild/tools/module_naming_scheme/__init__.py index bea5d52dc5..e54f04c31e 100644 --- a/easybuild/tools/module_naming_scheme/__init__.py +++ b/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/categorized_mns.py b/easybuild/tools/module_naming_scheme/categorized_mns.py index 616baf19c2..cdafc97cbb 100644 --- a/easybuild/tools/module_naming_scheme/categorized_mns.py +++ b/easybuild/tools/module_naming_scheme/categorized_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/easybuild_mns.py b/easybuild/tools/module_naming_scheme/easybuild_mns.py index c12ae9a614..93ff663e77 100644 --- a/easybuild/tools/module_naming_scheme/easybuild_mns.py +++ b/easybuild/tools/module_naming_scheme/easybuild_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index 9f2e026eb8..9332738b26 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py index 8d0c34297f..5603b7db47 100644 --- a/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py +++ b/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 487dbac063..fd60dbbe90 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 135d996ffb..b85032e6e6 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/module_naming_scheme/utilities.py b/easybuild/tools/module_naming_scheme/utilities.py index e46609d22b..018259ea4d 100644 --- a/easybuild/tools/module_naming_scheme/utilities.py +++ b/easybuild/tools/module_naming_scheme/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 8df257fa90..cd56888803 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/multidiff.py b/easybuild/tools/multidiff.py index 0f8c7f5bac..99ee949e56 100644 --- a/easybuild/tools/multidiff.py +++ b/easybuild/tools/multidiff.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 0ea04c7672..1ce8abee30 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index 0dc5b3b362..b6da8a9245 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # # -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py index 9ae9b37419..033687fac1 100644 --- a/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py +++ b/easybuild/tools/package/package_naming_scheme/easybuild_deb_friendly_pns.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- vim: set fileencoding=utf-8 ## -# Copyright 2015-2022 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py index a6543a7d53..7a53a86101 100644 --- a/easybuild/tools/package/package_naming_scheme/easybuild_pns.py +++ b/easybuild/tools/package/package_naming_scheme/easybuild_pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/package_naming_scheme/pns.py b/easybuild/tools/package/package_naming_scheme/pns.py index 081a94b073..8683ae8d37 100644 --- a/easybuild/tools/package/package_naming_scheme/pns.py +++ b/easybuild/tools/package/package_naming_scheme/pns.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 087084c687..43b7828850 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 6c94517c69..3ab31e1efc 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/__init__.py b/easybuild/tools/py2vs3/__init__.py index 5d9854b39f..5d45e11dc3 100644 --- a/easybuild/tools/py2vs3/__init__.py +++ b/easybuild/tools/py2vs3/__init__.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 1fec4650b1..f619279060 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index 8d9145179d..62e7a93723 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -1,5 +1,5 @@ # -# Copyright 2019-2023 Ghent University +# Copyright 2019-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index c7d0cb366d..b18c3c6159 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index 97fc6c33da..feaf8cb2ad 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py index 8cdf145b52..83a7890cc3 100644 --- a/easybuild/tools/repository/hgrepo.py +++ b/easybuild/tools/repository/hgrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/repository.py b/easybuild/tools/repository/repository.py index cbd33ab654..4948c461ac 100644 --- a/easybuild/tools/repository/repository.py +++ b/easybuild/tools/repository/repository.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 6c955c7fd0..79f593b1ee 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 68a4c9028b..b851b072ef 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0685fb37f7..30f413be55 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 59b8728454..73eb657510 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index ffd8ce580b..bc7817721f 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/__init__.py b/easybuild/tools/toolchain/__init__.py index 876d16c44f..cf755a8e40 100644 --- a/easybuild/tools/toolchain/__init__.py +++ b/easybuild/tools/toolchain/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 5d9f025172..faf8f03d88 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/constants.py b/easybuild/tools/toolchain/constants.py index 5655f71961..643e8b4d21 100644 --- a/easybuild/tools/toolchain/constants.py +++ b/easybuild/tools/toolchain/constants.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/fft.py b/easybuild/tools/toolchain/fft.py index c0e1dedd1a..b7191d5a26 100644 --- a/easybuild/tools/toolchain/fft.py +++ b/easybuild/tools/toolchain/fft.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 907993571c..fd74615c4a 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 37c964907a..0ea2714490 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 06dcf9ee5d..391a2831e0 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 4c30faf19a..409aac1f6c 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/toolchainvariables.py b/easybuild/tools/toolchain/toolchainvariables.py index 286b32aea3..dbb5686040 100644 --- a/easybuild/tools/toolchain/toolchainvariables.py +++ b/easybuild/tools/toolchain/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 56242b8fe4..8047778391 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/toolchain/variables.py b/easybuild/tools/toolchain/variables.py index c201f8a6f8..482f989480 100644 --- a/easybuild/tools/toolchain/variables.py +++ b/easybuild/tools/toolchain/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index b879a4005c..8f6407e55d 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index e5fa534c45..cef771f3d8 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index d296c6112d..006bb5543e 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/eb b/eb index d214d82b34..ad14ba61d0 100755 --- a/eb +++ b/eb @@ -1,6 +1,6 @@ #!/bin/bash ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/setup.py b/setup.py index 72ffd0e19a..82bae3aa2f 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/__init__.py b/test/__init__.py index 8793b2e4e1..20c9130dcc 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/__init__.py b/test/framework/__init__.py index c6a455e689..fae73ca9e6 100644 --- a/test/framework/__init__.py +++ b/test/framework/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/asyncprocess.py b/test/framework/asyncprocess.py index f8db1927d8..c7eced6c48 100644 --- a/test/framework/asyncprocess.py +++ b/test/framework/asyncprocess.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/build_log.py b/test/framework/build_log.py index a1792b998f..15f6984099 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/config.py b/test/framework/config.py index 4b42672cfc..55234d31b9 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/containers.py b/test/framework/containers.py index bd6dfde7f6..1d9abbf8e4 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2023 Ghent University +# Copyright 2018-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/docs.py b/test/framework/docs.py index 70280892e4..6b5b68ad17 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 6c08f947d9..e6b54e0bc8 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 905cf6c15b..0112fd3c08 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigformat.py b/test/framework/easyconfigformat.py index e806a88378..7b84a2c867 100644 --- a/test/framework/easyconfigformat.py +++ b/test/framework/easyconfigformat.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigparser.py b/test/framework/easyconfigparser.py index 9a17d8f0a5..fa943df938 100644 --- a/test/framework/easyconfigparser.py +++ b/test/framework/easyconfigparser.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easyconfigversion.py b/test/framework/easyconfigversion.py index 5ef706a72a..4da54b450a 100644 --- a/test/framework/easyconfigversion.py +++ b/test/framework/easyconfigversion.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/easystack.py b/test/framework/easystack.py index 91efcc94b3..a51059d280 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/ebconfigobj.py b/test/framework/ebconfigobj.py index d4d81995d8..169fe988a2 100644 --- a/test/framework/ebconfigobj.py +++ b/test/framework/ebconfigobj.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/environment.py b/test/framework/environment.py index 9a81e17486..5e90250d29 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8ee6b2c917..405ffc6b93 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/format_convert.py b/test/framework/format_convert.py index 1bc3f764bb..e0add188b3 100644 --- a/test/framework/format_convert.py +++ b/test/framework/format_convert.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/general.py b/test/framework/general.py index 526fe07467..3837cecc87 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/github.py b/test/framework/github.py index f011c89b4f..33d29ea8fa 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 152d2352a4..5e810ac0f9 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -1,5 +1,5 @@ # # -# Copyright 2017-2023 Ghent University +# Copyright 2017-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/include.py b/test/framework/include.py index b346f911ae..09a5925a2f 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/lib.py b/test/framework/lib.py index 2f4149314c..7930e00567 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -1,5 +1,5 @@ # # -# Copyright 2018-2023 Ghent University +# Copyright 2018-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/license.py b/test/framework/license.py index 80c741908f..6dc79ac6a4 100644 --- a/test/framework/license.py +++ b/test/framework/license.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index d2fbb6f642..456c18326e 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modules.py b/test/framework/modules.py index 2a05395b7c..6fab651546 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index f43a91e3b3..8dd67a4fdb 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/options.py b/test/framework/options.py index a491936d78..a083a8495c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/output.py b/test/framework/output.py index d1673a55d1..4429fe29ee 100644 --- a/test/framework/output.py +++ b/test/framework/output.py @@ -1,5 +1,5 @@ # # -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/package.py b/test/framework/package.py index 23e2e37e9c..b839fb32bb 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/parallelbuild.py b/test/framework/parallelbuild.py index 3ad85ac3b1..4e75672815 100644 --- a/test/framework/parallelbuild.py +++ b/test/framework/parallelbuild.py @@ -1,5 +1,5 @@ # # -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/repository.py b/test/framework/repository.py index aa1cf402b7..720e0fa8e8 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/robot.py b/test/framework/robot.py index 4f226bae35..bf936dcb45 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/run.py b/test/framework/run.py index d529b41a3a..bab4391cf6 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1,6 +1,6 @@ # # # -*- coding: utf-8 -*- -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py index e27c0c66d0..7d2ef2d6da 100644 --- a/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py +++ b/test/framework/sandbox/easybuild/easyblocks/e/easybuildmeta.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py index 194cead197..0871ce62e1 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/fftw.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foo.py b/test/framework/sandbox/easybuild/easyblocks/f/foo.py index 07e21608fa..b5a91d9503 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py index 3d64b1cf97..6415d6e6e1 100644 --- a/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py +++ b/test/framework/sandbox/easybuild/easyblocks/f/foofoo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py index a85b2482fa..5ee5aae1c6 100644 --- a/test/framework/sandbox/easybuild/easyblocks/g/gcc.py +++ b/test/framework/sandbox/easybuild/easyblocks/g/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py index d2faf66053..8ba475f700 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/bar.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/bar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py index 4ad690fe5c..b82668434c 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py index d754117b39..3466ce28f2 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py index 6b258c87d6..dd89704638 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/makecp.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py index 33ab7db67f..72252a46f4 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/modulerc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py index 0321602f3f..9c01951adf 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/pythonbundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2020 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py index fb81a51c96..5151221a13 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 9c700cf779..9335611f58 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py index 89fba75c30..db929d4d31 100644 --- a/test/framework/sandbox/easybuild/easyblocks/h/hpl.py +++ b/test/framework/sandbox/easybuild/easyblocks/h/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index 50b573649a..d3e55ed05d 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2023 Ghent University +# Copyright 2021-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py index bfb118cefa..e5dc093347 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openblas.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py index 630d5413a7..b110b4f921 100644 --- a/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py +++ b/test/framework/sandbox/easybuild/easyblocks/o/openmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py index d404f81648..3ded1247f4 100644 --- a/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py +++ b/test/framework/sandbox/easybuild/easyblocks/s/scalapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index c4614e3333..04186a37ef 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 3695744630..6445655707 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py index 0b6820adc9..09442d1a79 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_eula.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2023 Ghent University +# Copyright 2020-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py index a1fc62c69b..a3ef746d53 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toytoy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/__init__.py b/test/framework/sandbox/easybuild/tools/__init__.py index 1fbb2401b7..389c5ff367 100644 --- a/test/framework/sandbox/easybuild/tools/__init__.py +++ b/test/framework/sandbox/easybuild/tools/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2023 Ghent University +# Copyright 2009-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py index 3a9383c79f..a519340298 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2011-2023 Ghent University +# Copyright 2011-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py index 4d33d3d1e7..9113d89045 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/broken_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 5d8dec26c8..424fb55d43 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index 3cbb5cb60d..a0e0429bfe 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/style.py b/test/framework/style.py index 298b165c24..bfdf4e026a 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2023 Ghent University +# Copyright 2016-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/suite.py b/test/framework/suite.py index 6a40e35754..3b8193469c 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -1,6 +1,6 @@ #!/usr/bin/python # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 45f51991ff..deece97702 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index aacde3baf7..91f4106a3e 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toolchainvariables.py b/test/framework/toolchainvariables.py index db90378d8b..e61bb9031c 100644 --- a/test/framework/toolchainvariables.py +++ b/test/framework/toolchainvariables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 0349652b47..b699eb37ba 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2013-2023 Ghent University +# Copyright 2013-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/tweak.py b/test/framework/tweak.py index eda8046d52..24ec85b2c8 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2023 Ghent University +# Copyright 2014-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index 83f1ef41ac..8b7edd3215 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 37a9b29318..3e60b45811 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index ba4766f302..a17d7beb02 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/variables.py b/test/framework/variables.py index 8331a6c2dd..cc804d6a65 100644 --- a/test/framework/variables.py +++ b/test/framework/variables.py @@ -1,5 +1,5 @@ # # -# Copyright 2012-2023 Ghent University +# Copyright 2012-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/test/framework/yeb.py b/test/framework/yeb.py index 74a2a47875..346ffa057b 100644 --- a/test/framework/yeb.py +++ b/test/framework/yeb.py @@ -1,5 +1,5 @@ # # -# Copyright 2015-2023 Ghent University +# Copyright 2015-2024 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), From 717f55576974348aca7e96475aeff62b29a8c187 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 12:07:02 +0200 Subject: [PATCH 153/171] implement qa_wait_patterns option in run_shell_cmd --- easybuild/tools/run.py | 22 +++++++++++++++++----- test/framework/run.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index ba40b704ac..4376fb0220 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -252,9 +252,8 @@ def to_cmd_str(cmd): if not isinstance(qa_patterns, list) or any(not isinstance(x, tuple) or len(x) != 2 for x in qa_patterns): raise EasyBuildError("qa_patterns passed to run_shell_cmd should be a list of 2-tuples!") - # temporarily raise a NotImplementedError until all options are implemented - if qa_wait_patterns: - raise NotImplementedError + if qa_wait_patterns is None: + qa_wait_patterns = [] if work_dir is None: work_dir = os.getcwd() @@ -377,10 +376,10 @@ def to_cmd_str(cmd): if split_stderr: stderr += proc.stderr.read1(read_size) or b'' - # only consider answering questions if there's new output beyond additional whitespace if qa_patterns: + match_found = False for question, answers in qa_patterns: - + # allow extra whitespace at the end question += r'[\s\n]*$' regex = re.compile(question.encode()) if regex.search(stdout): @@ -397,8 +396,21 @@ def to_cmd_str(cmd): answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 + match_found = True break else: + # if no match was found among question patterns, + # take into account patterns for non-questions (qa_wait_patterns) + for pattern in qa_wait_patterns: + # allow extra whitespace at the end + pattern += r'[\s\n]*$' + regex = re.compile(pattern.encode()) + if regex.search(stdout): + time_no_match = 0 + match_found = True + break + + if not match_found: # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs if time_no_match > qa_timeout: diff --git a/test/framework/run.py b/test/framework/run.py index 6fb8d2b072..ba9a262925 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -771,6 +771,24 @@ def test_run_cmd_qa(self): self.assertTrue(out.startswith("question\nanswer\nfoo ")) self.assertTrue(out.endswith('bar')) + # test handling of output that is not actually a question + cmd = ';'.join([ + "echo not-a-question-but-a-statement", + "sleep 3", + "echo question", + "read x", + "echo $x", + ]) + qa = {'question': 'answer'} + + # fails because non-question is encountered + error_pattern = "Max nohits 1 reached: end of output not-a-question-but-a-statement" + self.assertErrorRegex(EasyBuildError, error_pattern, run_cmd_qa, cmd, qa, maxhits=1, trace=False) + + (out, ec) = run_cmd_qa(cmd, qa, no_qa=["not-a-question-but-a-statement"], maxhits=1, trace=False) + self.assertEqual(out, "not-a-question-but-a-statement\nquestion\nanswer\n") + self.assertEqual(ec, 0) + def test_run_shell_cmd_qa(self): """Basic test for Q&A support in run_shell_cmd function.""" @@ -820,6 +838,27 @@ def test_run_shell_cmd_qa(self): with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1) + # test handling of output that is not actually a question + cmd = ';'.join([ + "echo not-a-question-but-a-statement", + "sleep 3", + "echo question", + "read x", + "echo $x", + ]) + qa = [('question', 'answer')] + + # fails because non-question is encountered + error_pattern = "No matching questions found for current command output, giving up after 1 seconds!" + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1, + hidden=True) + + qa_wait_patterns = ["not-a-question-but-a-statement"] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=qa_wait_patterns, qa_timeout=1) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "not-a-question-but-a-statement\nquestion\nanswer\n") + def test_run_cmd_qa_buffering(self): """Test whether run_cmd_qa uses unbuffered output.""" From f020bf901f71c6a3bad2b65318c751deb9e71c51 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 12:08:11 +0200 Subject: [PATCH 154/171] fix excessively long line in test_run_shell_cmd_qa_answers --- test/framework/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index ba9a262925..0000a8efbd 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -1047,7 +1047,8 @@ def test_run_shell_cmd_qa_answers(self): self.assertEqual(res.exit_code, 0) with self.mocked_stdout_stderr(): - self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd, qa_patterns=[('question', 1)]) + self.assertErrorRegex(EasyBuildError, "Unknown type of answers encountered", run_shell_cmd, cmd, + qa_patterns=[('question', 1)]) # test cycling of answers cmd = cmd * 2 From 7d5c7ce51c9a4a852082df743dbbd4c90171a546 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 12:21:58 +0200 Subject: [PATCH 155/171] add support to run_shell_cmd for completing answer via pattern extracted from question --- easybuild/tools/run.py | 5 ++++- test/framework/run.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 4376fb0220..7fffe2ba6f 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -382,7 +382,8 @@ def to_cmd_str(cmd): # allow extra whitespace at the end question += r'[\s\n]*$' regex = re.compile(question.encode()) - if regex.search(stdout): + res = regex.search(stdout) + if res: # if answer is specified as a list, we take the first item as current answer, # and add it to the back of the list (so we cycle through answers) if isinstance(answers, list): @@ -393,6 +394,8 @@ def to_cmd_str(cmd): else: raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + # answer may need to be completed via pattern extracted from question + answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} answer += '\n' os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 diff --git a/test/framework/run.py b/test/framework/run.py index 0000a8efbd..6ec6ebfb10 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -838,6 +838,18 @@ def test_run_shell_cmd_qa(self): with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, cmd, qa_patterns=qa, qa_timeout=1) + # check using answer that is completed via pattern extracted from question + cmd = ';'.join([ + "echo 'and the magic number is: 42'", + "read magic_number", + "echo $magic_number", + ]) + qa = [("and the magic number is: (?P[0-9]+)", "%(nr)s")] + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, qa_patterns=qa) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "and the magic number is: 42\n42\n") + # test handling of output that is not actually a question cmd = ';'.join([ "echo not-a-question-but-a-statement", From 110f41c103f3e11fe7e031f48d889c021cbe9ea4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 15:41:52 +0200 Subject: [PATCH 156/171] fix comment in test_fetch_files_from_commit (not testing with short commit ID) Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- test/framework/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/github.py b/test/framework/github.py index 4bcf9b8987..e2bc20daf7 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -540,7 +540,7 @@ def test_github_fetch_files_from_pr_cache(self): def test_fetch_files_from_commit(self): """Test fetch_files_from_commit function.""" - # easyconfigs commit to add EasyBuild-4.8.2.eb (also test short commit, should work) + # easyconfigs commit to add EasyBuild-4.8.2.eb test_commit = '7c83a553950c233943c7b0189762f8c05cfea852' # without specifying any files/repo, default is to use easybuilders/easybuilld-easyconfigs From b9f6f51e87433694bf76177ccac577eea14604fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 15:42:16 +0200 Subject: [PATCH 157/171] mention 'commit' option in docstring for download_repo --- easybuild/tools/github.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index eda237fca1..bd80c3c0e8 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -361,6 +361,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch=None, commit=None, accoun Download entire GitHub repo as a tar.gz archive, and extract it into specified path. :param repo: repo to download :param branch: branch to download + :param commit: commit to download :param account: GitHub account to download repo from :param path: path to extract to :param github_user: name of GitHub user to use From 5f169a9f7845cef045a2365f9a25cbd452cb40fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 15:44:01 +0200 Subject: [PATCH 158/171] GitHub token is not required in test_github_det_easyconfig_paths_from_commit --- test/framework/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/robot.py b/test/framework/robot.py index ccdf9a2b1e..680d2a70bc 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -715,6 +715,7 @@ def test_search_paths(self): def test_github_det_easyconfig_paths_from_commit(self): """Test det_easyconfig_paths function in combination with --from-commit.""" + # note: --from-commit does not involve using GitHub API, so no GitHub token required test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -730,7 +731,6 @@ def test_github_det_easyconfig_paths_from_commit(self): '--robot', '--robot=%s' % test_ecs_path, '--unittest-file=%s' % self.logfile, - '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user '--tmpdir=%s' % self.test_prefix, ] From 7145ca9676404eba51cef4e815b7c5745209f569 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 16:40:25 +0200 Subject: [PATCH 159/171] improve logging in Q&A part of run_shell_cmd --- easybuild/tools/run.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7fffe2ba6f..26f2deed5c 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -286,12 +286,14 @@ def to_cmd_str(cmd): else: cmd_out_fp, cmd_err_fp = None, None + interactive = bool(qa_patterns) + interactive_msg = 'interactive ' if interactive else '' + # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) if not in_dry_run and build_option('extended_dry_run'): if not hidden or verbose_dry_run: silent = build_option('silent') - interactive = 'interactive ' if qa_patterns else '' - msg = f" running {interactive}shell command \"{cmd_str}\"\n" + msg = f" running {interactive_msg}shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) @@ -300,8 +302,7 @@ def to_cmd_str(cmd): start_time = datetime.now() if not hidden: - _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, - interactive=bool(qa_patterns)) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id, interactive=interactive) if stream_output: print_msg(f"(streaming) output for command '{cmd_str}':") @@ -319,7 +320,7 @@ def to_cmd_str(cmd): if with_hooks: hooks = load_hooks(build_option('hooks')) kwargs = { - 'interactive': bool(qa_patterns), + 'interactive': interactive, 'work_dir': work_dir, } hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs) @@ -330,7 +331,7 @@ def to_cmd_str(cmd): stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - log_msg = f"Running shell command '{cmd_str}' in {work_dir}" + log_msg = f"Running {interactive_msg}shell command '{cmd_str}' in {work_dir}" if thread_id: log_msg += f" (via thread with ID {thread_id})" _log.info(log_msg) @@ -384,6 +385,7 @@ def to_cmd_str(cmd): regex = re.compile(question.encode()) res = regex.search(stdout) if res: + _log.debug(f"Found match for question pattern '{question}' at end of: {stdout}") # if answer is specified as a list, we take the first item as current answer, # and add it to the back of the list (so we cycle through answers) if isinstance(answers, list): @@ -395,13 +397,16 @@ def to_cmd_str(cmd): raise EasyBuildError(f"Unknown type of answers encountered: {answers}") # answer may need to be completed via pattern extracted from question + _log.debug(f"Raw answer for question pattern '{question}': {answer}") answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} answer += '\n' + _log.info(f"Found match for question pattern '{question}', replying with: {answer}") os.write(proc.stdin.fileno(), answer.encode()) time_no_match = 0 match_found = True break else: + _log.info("No match found for question patterns, considering question wait patterns") # if no match was found among question patterns, # take into account patterns for non-questions (qa_wait_patterns) for pattern in qa_wait_patterns: @@ -409,17 +414,24 @@ def to_cmd_str(cmd): pattern += r'[\s\n]*$' regex = re.compile(pattern.encode()) if regex.search(stdout): + _log.info(f"Found match for question wait pattern '{pattern}'") + _log.debug(f"Found match for question wait pattern '{pattern}' at end of: {stdout}") time_no_match = 0 match_found = True break + else: + _log.info("No match found for question wait patterns") if not match_found: + _log.debug(f"No match found in question or wait patterns at end of current output: {stdout}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs if time_no_match > qa_timeout: error_msg = "No matching questions found for current command output, " error_msg += f"giving up after {qa_timeout} seconds!" raise EasyBuildError(error_msg) + else: + _log.debug(f"{time_no_match} seconds without match in output of interactive shell command") time.sleep(check_interval_secs) @@ -467,7 +479,7 @@ def to_cmd_str(cmd): if with_hooks: run_hook_kwargs = { 'exit_code': res.exit_code, - 'interactive': bool(qa_patterns), + 'interactive': interactive, 'output': res.output, 'stderr': res.stderr, 'work_dir': res.work_dir, From f3e5a650d7dacd52e72c052dbe963b2483280b00 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 17:09:56 +0200 Subject: [PATCH 160/171] only show last 1000 characters in debug log message when answering questions for interactive commands in run_shell_cmd function --- easybuild/tools/run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 26f2deed5c..b63852b87c 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -385,7 +385,7 @@ def to_cmd_str(cmd): regex = re.compile(question.encode()) res = regex.search(stdout) if res: - _log.debug(f"Found match for question pattern '{question}' at end of: {stdout}") + _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout[:1000]}") # if answer is specified as a list, we take the first item as current answer, # and add it to the back of the list (so we cycle through answers) if isinstance(answers, list): @@ -414,8 +414,8 @@ def to_cmd_str(cmd): pattern += r'[\s\n]*$' regex = re.compile(pattern.encode()) if regex.search(stdout): - _log.info(f"Found match for question wait pattern '{pattern}'") - _log.debug(f"Found match for question wait pattern '{pattern}' at end of: {stdout}") + _log.info(f"Found match for wait pattern '{pattern}'") + _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") time_no_match = 0 match_found = True break @@ -423,7 +423,7 @@ def to_cmd_str(cmd): _log.info("No match found for question wait patterns") if not match_found: - _log.debug(f"No match found in question or wait patterns at end of current output: {stdout}") + _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout[:1000]}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs if time_no_match > qa_timeout: From ca21ff655675c0a28bbbf41a47d515ba93be4c4b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 18:13:45 +0200 Subject: [PATCH 161/171] only check for questions to be answered every 0.1s --- easybuild/tools/run.py | 4 ++-- test/framework/run.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index b63852b87c..bb26dc9482 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -360,7 +360,7 @@ def to_cmd_str(cmd): exit_code = None stdout, stderr = b'', b'' - check_interval_secs = 0.001 + check_interval_secs = 0.1 time_no_match = 0 # collect output piece-wise, while checking for questions to answer (if qa_patterns is provided) @@ -431,7 +431,7 @@ def to_cmd_str(cmd): error_msg += f"giving up after {qa_timeout} seconds!" raise EasyBuildError(error_msg) else: - _log.debug(f"{time_no_match} seconds without match in output of interactive shell command") + _log.debug(f"{time_no_match:0.1f} seconds without match in output of interactive shell command") time.sleep(check_interval_secs) diff --git a/test/framework/run.py b/test/framework/run.py index 6ec6ebfb10..6e64764805 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -907,8 +907,8 @@ def test_run_shell_cmd_qa_buffering(self): """Test whether run_shell_cmd uses unbuffered output when running interactive commands.""" # command that generates a lot of output before waiting for input - # note: bug being fixed can be reproduced reliably using 1000, but not with too high values like 100000! - cmd = 'for x in $(seq 1000); do echo "This is a number you can pick: $x"; done; ' + # note: bug being fixed can be reproduced reliably using 100, but not with too high values like 100000! + cmd = 'for x in $(seq 100); do echo "This is a number you can pick: $x"; done; ' cmd += 'echo "Pick a number: "; read number; echo "Picked number: $number"' with self.mocked_stdout_stderr(): res = run_shell_cmd(cmd, qa_patterns=[('Pick a number: ', '42')], qa_timeout=10) From 2abb706a839e49c022a2b621f572d6fae2e686eb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 18:26:23 +0200 Subject: [PATCH 162/171] flesh out code to try and answer question raised by interactive shell command from run_shell_cmd into private helper function _answer_question --- easybuild/tools/run.py | 104 +++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 45 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index bb26dc9482..24e860054b 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -202,6 +202,63 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) +def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): + """ + Private helper function to try and answer questions raised in interactive shell commands. + """ + match_found = False + + for question, answers in qa_patterns: + # allow extra whitespace at the end + question += r'[\s\n]*$' + regex = re.compile(question.encode()) + res = regex.search(stdout) + if res: + _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout[:1000]}") + # if answer is specified as a list, we take the first item as current answer, + # and add it to the back of the list (so we cycle through answers) + if isinstance(answers, list): + answer = answers.pop(0) + answers.append(answer) + elif isinstance(answers, str): + answer = answers + else: + raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + + # answer may need to be completed via pattern extracted from question + _log.debug(f"Raw answer for question pattern '{question}': {answer}") + answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} + answer += '\n' + _log.info(f"Found match for question pattern '{question}', replying with: {answer}") + + try: + os.write(proc.stdin.fileno(), answer.encode()) + except OSError as err: + raise EasyBuildError("Failed to answer question raised by interactive command: %s", err) + + time_no_match = 0 + match_found = True + break + else: + _log.info("No match found for question patterns, considering question wait patterns") + # if no match was found among question patterns, + # take into account patterns for non-questions (qa_wait_patterns) + for pattern in qa_wait_patterns: + # allow extra whitespace at the end + pattern += r'[\s\n]*$' + regex = re.compile(pattern.encode()) + if regex.search(stdout): + _log.info(f"Found match for wait pattern '{pattern}'") + _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") + time_no_match = 0 + match_found = True + break + else: + _log.info("No match found for question wait patterns") + + return match_found + + @run_shell_cmd_cache def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, @@ -378,51 +435,7 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' if qa_patterns: - match_found = False - for question, answers in qa_patterns: - # allow extra whitespace at the end - question += r'[\s\n]*$' - regex = re.compile(question.encode()) - res = regex.search(stdout) - if res: - _log.debug(f"Found match for question pattern '{question}' at end of stdout: {stdout[:1000]}") - # if answer is specified as a list, we take the first item as current answer, - # and add it to the back of the list (so we cycle through answers) - if isinstance(answers, list): - answer = answers.pop(0) - answers.append(answer) - elif isinstance(answers, str): - answer = answers - else: - raise EasyBuildError(f"Unknown type of answers encountered: {answers}") - - # answer may need to be completed via pattern extracted from question - _log.debug(f"Raw answer for question pattern '{question}': {answer}") - answer = answer % {k: v.decode() for (k, v) in res.groupdict().items()} - answer += '\n' - _log.info(f"Found match for question pattern '{question}', replying with: {answer}") - os.write(proc.stdin.fileno(), answer.encode()) - time_no_match = 0 - match_found = True - break - else: - _log.info("No match found for question patterns, considering question wait patterns") - # if no match was found among question patterns, - # take into account patterns for non-questions (qa_wait_patterns) - for pattern in qa_wait_patterns: - # allow extra whitespace at the end - pattern += r'[\s\n]*$' - regex = re.compile(pattern.encode()) - if regex.search(stdout): - _log.info(f"Found match for wait pattern '{pattern}'") - _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") - time_no_match = 0 - match_found = True - break - else: - _log.info("No match found for question wait patterns") - - if not match_found: + if not _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout[:1000]}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs @@ -437,6 +450,7 @@ def to_cmd_str(cmd): exit_code = proc.poll() + # collect last bit of output once processed has exited stdout += proc.stdout.read() if split_stderr: stderr += proc.stderr.read() From 23727d605563ce760a82ea4ba1fc9bc2b7306bad Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 18:27:38 +0200 Subject: [PATCH 163/171] don't reset time_no_match in _answer_question, do it in run_shell_cmd --- easybuild/tools/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 24e860054b..c1d13d01d0 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -236,7 +236,6 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): except OSError as err: raise EasyBuildError("Failed to answer question raised by interactive command: %s", err) - time_no_match = 0 match_found = True break else: @@ -250,7 +249,6 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): if regex.search(stdout): _log.info(f"Found match for wait pattern '{pattern}'") _log.debug(f"Found match for wait pattern '{pattern}' at end of stdout: {stdout[:1000]}") - time_no_match = 0 match_found = True break else: @@ -435,7 +433,9 @@ def to_cmd_str(cmd): stderr += proc.stderr.read1(read_size) or b'' if qa_patterns: - if not _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): + if _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): + time_no_match = 0 + else: _log.debug(f"No match found in question/wait patterns at end of stdout: {stdout[:1000]}") # this will only run if the for loop above was *not* stopped by the break statement time_no_match += check_interval_secs From 6968f87da75b1e3b54c6bb6ee39e21d22f94f192 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 21:03:52 +0200 Subject: [PATCH 164/171] raise error when --from-commit and --from-pr or --include-easyblocks-from-commit and --include-easyblocks-from-pr are combined --- easybuild/main.py | 8 +++++++- easybuild/tools/options.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 4ca7bcbb95..8ffb185061 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -765,8 +765,14 @@ def prepare_main(args=None, logfile=None, testing=None): if __name__ == "__main__": - init_session_state, eb_go, cfg_settings = prepare_main() + # take into account that EasyBuildError may be raised when parsing the EasyBuild configuration + try: + init_session_state, eb_go, cfg_settings = prepare_main() + except EasyBuildError as err: + print_error(err.msg) + hooks = load_hooks(eb_go.options.hooks) + try: main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 91bede9553..9eb6d915e5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1512,6 +1512,19 @@ def parse_options(args=None, with_include=True): return eb_go +def check_options(options): + """ + Check configuration options, some combinations are not allowed. + """ + if options.from_commit and options.from_pr: + raise EasyBuildError("--from-commit and --from-pr should not be used together, pick one") + + if options.include_easyblocks_from_commit and options.include_easyblocks_from_pr: + error_msg = "--include-easyblocks-from-commit and --include-easyblocks-from-pr " + error_msg += "should not be used together, pick one" + raise EasyBuildError(error_msg) + + def check_root_usage(allow_use_as_root=False): """ Check whether we are running as root, and act accordingly @@ -1613,6 +1626,8 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False, r eb_go = parse_options(args=args) options = eb_go.options + check_options(options) + # tmpdir is set by option parser via set_tmpdir function tmpdir = tempfile.gettempdir() From 52648c09a33c17885ee0fb73847dbe9ef2aadb2a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 21:16:19 +0200 Subject: [PATCH 165/171] fix indentation in handle_include_easyblocks_from that breaks --include-easyblocks-from-pr if --include-easyblocks is not used (cfr. failing test_github_xxx_include_easyblocks_from_pr) --- easybuild/tools/options.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9eb6d915e5..a568c6263e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1574,8 +1574,7 @@ def check_included_multiple(included_easyblocks_from, source): if options.include_easyblocks: check_included_multiple(included_from_pr, "PR #%s" % easyblock_pr) - - included_easyblocks |= included_from_pr + included_easyblocks |= included_from_pr for easyblock in included_from_pr: print_msg("easyblock %s included from PR #%s" % (easyblock, easyblock_pr), log=log) From 640142adcefc164ac91ef51f2072cf59da1ceda8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Apr 2024 21:37:01 +0200 Subject: [PATCH 166/171] add comment to clarify why there's no early stop when --terse is combined with --missing-modules --- easybuild/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index 826880aad1..caaa4695e3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -684,6 +684,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr options.list_prs, options.merge_pr, options.review_pr, + # --missing-modules is processed by process_eb_args, + # so we can't exit just yet here if it's used in combination with --terse options.terse and not options.missing_modules, search_query, ] From b65692dc6a7a992eecdea786423682a72e016247 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 4 Apr 2024 10:18:43 +0800 Subject: [PATCH 167/171] prepare release notes for EasyBuild v4.9.1 + bump version to 4.9.1 --- RELEASE_NOTES | 35 +++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 94b66db903..1d13b3a121 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,41 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.1 (5 April 2024) +--------------------- + +update/bugfix release + +- various enhancements, including: + - make `is_rpath_wrapper` faster by only checking file contents if file is not located in subdirectory of RPATH wrapper subdirectory (#4406) + - add terse support to `--missing-modules` (#4407) + - adapt version pattern for EnvironmentModules to allow using development version (#4416) + - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417) + - add support for appending to path environment variables via `modextrapaths_append` (and add corresponding `allow_append_abs_path`) (#4436) + - improve output produced by `--check-github` (#4437) + - add script for updating local git repos with develop branch (#4438) + - show error when multiple PR options are passed (#4440) + - improve `findPythonDeps` script to recognize non-canonical package names (#4445) + - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468) + - improve logging & handling of (empty) `--optarch` values (#4481) + - add `--short` option to findUpdatedEcs (#4488) + - add generic GCC and Clang compiler flags for RISC-V (#4489) +- various bug fixes, including: + - clean up log file of EasyBlock instance in `check_sha256_checksums` (#4452) + - fix description of `backup-modules` configuration option (#4456) + - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable (#4461) + - use `cp -dR` instead of `cp -a` for shell script extraction (#4465) + - fix link to documentation in `close_pr` message (#4466) + - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470) + - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472) +- other changes: + - clean up & speed up environment checks (#4409) + - use more performant and concise dict construction by using dict comprehensions (#4410) + - remove superflous string formatting (#4411) + - clean up uses of `getattr` and `hasattr` (#4412) + - update copyright lines to 2024 (#4494) + + v4.9.0 (30 December 2023) ------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 006bb5543e..0e82ff2ce9 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.1.dev0') +VERSION = LooseVersion('4.9.1') UNKNOWN = 'UNKNOWN' From cd1a860e5453c81bf4b25b7c9af40365d2d710bb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Apr 2024 17:23:47 +0200 Subject: [PATCH 168/171] minor tweaks to 4.9.1 release notes --- RELEASE_NOTES | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 1d13b3a121..07c9756e67 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -14,20 +14,20 @@ update/bugfix release - add terse support to `--missing-modules` (#4407) - adapt version pattern for EnvironmentModules to allow using development version (#4416) - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417) - - add support for appending to path environment variables via `modextrapaths_append` (and add corresponding `allow_append_abs_path`) (#4436) + - add support for appending to path environment variables via `modextrapaths_append` + add corresponding `allow_append_abs_path` (#4436) - improve output produced by `--check-github` (#4437) - - add script for updating local git repos with develop branch (#4438) + - add script for updating local git repos with `develop` branch (#4438) - show error when multiple PR options are passed (#4440) - improve `findPythonDeps` script to recognize non-canonical package names (#4445) - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468) - improve logging & handling of (empty) `--optarch` values (#4481) - - add `--short` option to findUpdatedEcs (#4488) + - add `--short` option to `findUpdatedEcs` script (#4488) - add generic GCC and Clang compiler flags for RISC-V (#4489) - various bug fixes, including: - - clean up log file of EasyBlock instance in `check_sha256_checksums` (#4452) + - clean up log file of `EasyBlock` instance in `check_sha256_checksums` (#4452) - fix description of `backup-modules` configuration option (#4456) - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable (#4461) - - use `cp -dR` instead of `cp -a` for shell script extraction (#4465) + - use `cp -dR` instead of `cp -a` for shell script "extraction" (#4465) - fix link to documentation in `close_pr` message (#4466) - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470) - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472) From ee73024d9c89ce4d528b56b1c29a9ed192daa03b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Apr 2024 20:21:21 +0200 Subject: [PATCH 169/171] bump version to 4.9.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 0e82ff2ce9..c01c8fc5c3 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.1') +VERSION = LooseVersion('4.9.2.dev0') UNKNOWN = 'UNKNOWN' From 204f3287b10974ba875337152ff0985fd3ae0083 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 10:19:47 +0200 Subject: [PATCH 170/171] tackle Simon's suggestions for run_shell_cmd --- easybuild/tools/run.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c1d13d01d0..3253a372b1 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -223,7 +223,7 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): elif isinstance(answers, str): answer = answers else: - raise EasyBuildError(f"Unknown type of answers encountered: {answers}") + raise EasyBuildError(f"Unknown type of answers encountered for question ({question}): {answers}") # answer may need to be completed via pattern extracted from question _log.debug(f"Raw answer for question pattern '{question}': {answer}") @@ -280,8 +280,7 @@ def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=N :param task_id: task ID for specified shell command (included in return value) :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers - :param qa_wait_patterns: list of 2-tuples with patterns for non-questions - and number of iterations to allow these patterns to match with end out command output + :param qa_wait_patterns: list of strings with patterns for non-questions :param qa_timeout: amount of seconds to wait until more output is produced when there is no matching question :return: Named tuple with: From ce9a555a2f97737bfa1d0c263059de9eb17b823b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Apr 2024 13:13:00 +0200 Subject: [PATCH 171/171] disable trace for download_file used in fetch_files_from_commit --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 3bcc99b83c..d1b93c3ad7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -637,7 +637,7 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff') diff_fn = os.path.basename(diff_url) diff_filepath = os.path.join(path, diff_fn) - if download_file(diff_fn, diff_url, diff_filepath, forced=True): + if download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False): diff_txt = read_file(diff_filepath) _log.debug("Diff for commit %s:\n%s", commit, diff_txt)