From e9c8f7b3ea5a3e5f46e1d35f5a71f483fb6c3ebb Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Tue, 13 Sep 2022 09:41:18 -0400 Subject: [PATCH 01/14] Add an --extra-source-urls CLI option to fetch sources from additional URLs. Add the --extra-source-urls CLI option, a | separated list of URLs that EasyBuild will fetch sources from. It replaces the hard-coded EASYBUILD_SOURCES_URL, but keeps it as a default value. Uses the add_flex logic to prepend, append, or insert into the default list. Co-authored-by: Alexander Grund --- easybuild/base/generaloption.py | 8 +++++++- easybuild/framework/easyblock.py | 11 +++++++---- easybuild/tools/config.py | 2 ++ easybuild/tools/options.py | 8 ++++++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 90248fb0cd..291158a955 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -103,6 +103,11 @@ def what_str_list_tuple(name): sep = os.pathsep helpsep = 'pathsep' + elif name.startswith('url'): + # | is one of the only characters not in the grammar for URIs (RFC3986) + sep = '|' + helpsep = '|' + klass = None if name.endswith('list'): klass = list @@ -182,6 +187,7 @@ class ExtOption(CompleterOption): - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings - the path separator is OS-dependent + - urllist, urltuple: using '|', convert urlsep separated string in a list resp. tuple of strings """ EXTEND_SEPARATOR = ',' @@ -198,7 +204,7 @@ class ExtOption(CompleterOption): TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS - TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']] + TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']] TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST} TYPE_CHECKER.update(Option.TYPE_CHECKER) TYPES = tuple(TYPE_STRLIST + list(Option.TYPES)) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ce436f1841..704b0bebc8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -106,8 +106,6 @@ from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION -EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' - DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] @@ -897,8 +895,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - # add https://sources.easybuild.io as fallback source URL - source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location)) + # add extra-source-urls CLI as either a first check, or a fallback. + self.log.warning("[extra_source_urls] %s", build_option('extra_source_urls')) + for url in build_option("extra_source_urls"): + url += "/" + name_letter + "/" + location + self.log.warning("[extra_source_urls] url is %s", url) + source_urls.extend([url]) + self.log.warning("[extra_source_urls] %s", source_urls) mkdir(targetdir, parents=True) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6bec64764c..8268addd68 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -120,6 +120,7 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' +DEFAULT_EXTRA_SOURCE_URLS = ('https://sources.easybuild.io/',) # Filter these CUDA libraries by default from the RPATH sanity check. # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build # against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should @@ -215,6 +216,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'easyblock', 'envvars_user_modules', 'extra_modules', + 'extra_source_urls', 'filter_deps', 'filter_ecs', 'filter_env_vars', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index df10ec859e..bcd53a1563 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,10 +69,12 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS +from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS -from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE +from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR +from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN @@ -407,6 +409,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig", + "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[%sURL]' % '|'}), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From 18a9e29e2601423a01f401699253e37db3ab74cd Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Tue, 16 Jul 2024 21:00:25 -0400 Subject: [PATCH 02/14] Addressed review comments. --- easybuild/framework/easyblock.py | 8 +++----- easybuild/tools/options.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 704b0bebc8..24fe58da5b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -895,13 +895,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - # add extra-source-urls CLI as either a first check, or a fallback. - self.log.warning("[extra_source_urls] %s", build_option('extra_source_urls')) + # Insert --extra-source-urls command line option to the + # urls to try to download from. for url in build_option("extra_source_urls"): url += "/" + name_letter + "/" + location - self.log.warning("[extra_source_urls] url is %s", url) - source_urls.extend([url]) - self.log.warning("[extra_source_urls] %s", source_urls) + source_urls.append(url) mkdir(targetdir, parents=True) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index bcd53a1563..9d44391877 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -410,7 +410,7 @@ def override_options(self): 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig", - "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[%sURL]' % '|'}), + "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[|URL]'}), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " From f4ec1825c740410f667342caa3403a4e37657c33 Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Sun, 4 Aug 2024 15:15:21 -0400 Subject: [PATCH 03/14] Handle the case build_option('extra_source_urls') is empty in unit tests. --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 24fe58da5b..f1f857d887 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -897,7 +897,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No # Insert --extra-source-urls command line option to the # urls to try to download from. - for url in build_option("extra_source_urls"): + for url in build_option("extra_source_urls") or []: url += "/" + name_letter + "/" + location source_urls.append(url) From 281f9bafe390a8007a4062fcf7c2d6614dfcfb24 Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Tue, 13 Aug 2024 10:43:51 -0400 Subject: [PATCH 04/14] extra-sources-url doesn't seem to be configured with init_config, testing whether specifying it here fixes the tests. --- easybuild/framework/easyblock.py | 3 +-- test/framework/easyblock.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f1f857d887..dbc1de4107 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -895,8 +895,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - # Insert --extra-source-urls command line option to the - # urls to try to download from. + # Add additional URLs as configured. for url in build_option("extra_source_urls") or []: url += "/" + name_letter + "/" + location source_urls.append(url) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 19218eca6a..a505a849a3 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1824,7 +1824,7 @@ def test_fallback_source_url(self): """Check whether downloading from fallback source URL https://sources.easybuild.io works.""" # cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11951 - init_config(args=["--sourcepath=%s" % self.test_prefix]) + init_config(args=["--sourcepath=%s" % self.test_prefix, "--extra-source-urls=https://sources.easybuild.io/"]) udunits_ec = os.path.join(self.test_prefix, 'UDUNITS.eb') udunits_ec_txt = '\n'.join([ From 1f5542e586ad3661c56d7394786627f6661a5bc5 Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Tue, 13 Aug 2024 10:59:34 -0400 Subject: [PATCH 05/14] Moved extra_source_urls to the proper location in BUILD_OPTIONS_CMDLINE so that init_config has the right default value. Should fix CI tests. --- easybuild/tools/config.py | 4 +++- test/framework/easyblock.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 8268addd68..fcd934ccd7 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -216,7 +216,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'easyblock', 'envvars_user_modules', 'extra_modules', - 'extra_source_urls', 'filter_deps', 'filter_ecs', 'filter_env_vars', @@ -393,6 +392,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'defaultopt': [ 'default_opt_level', ], + DEFAULT_EXTRA_SOURCE_URLS: [ + 'extra_source_urls', + ], DEFAULT_ALLOW_LOADED_MODULES: [ 'allow_loaded_modules', ], diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index a505a849a3..19218eca6a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1824,7 +1824,7 @@ def test_fallback_source_url(self): """Check whether downloading from fallback source URL https://sources.easybuild.io works.""" # cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11951 - init_config(args=["--sourcepath=%s" % self.test_prefix, "--extra-source-urls=https://sources.easybuild.io/"]) + init_config(args=["--sourcepath=%s" % self.test_prefix]) udunits_ec = os.path.join(self.test_prefix, 'UDUNITS.eb') udunits_ec_txt = '\n'.join([ From d6a8d72f5c1d1b5f40f8ec03059779daf75bc3bc Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Wed, 14 Aug 2024 21:16:08 -0400 Subject: [PATCH 06/14] Addressed review comments. --- easybuild/base/generaloption.py | 8 ++++---- easybuild/framework/easyblock.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/options.py | 3 +-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 291158a955..c3660f7cbf 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -97,16 +97,16 @@ def what_str_list_tuple(name): """Given name, return separator, class and helptext wrt separator. (Currently supports strlist, strtuple, pathlist, pathtuple) """ - sep = ',' - helpsep = 'comma' if name.startswith('path'): sep = os.pathsep helpsep = 'pathsep' - elif name.startswith('url'): # | is one of the only characters not in the grammar for URIs (RFC3986) sep = '|' helpsep = '|' + else: + sep = ',' + helpsep = 'comma' klass = None if name.endswith('list'): @@ -187,7 +187,7 @@ class ExtOption(CompleterOption): - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings - the path separator is OS-dependent - - urllist, urltuple: using '|', convert urlsep separated string in a list resp. tuple of strings + - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings """ EXTEND_SEPARATOR = ',' diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dbc1de4107..a503318574 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -896,7 +896,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls.extend(self.cfg['source_urls']) # Add additional URLs as configured. - for url in build_option("extra_source_urls") or []: + for url in build_option("extra_source_urls"): url += "/" + name_letter + "/" + location source_urls.append(url) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index fcd934ccd7..e36e48179a 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -120,7 +120,7 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' -DEFAULT_EXTRA_SOURCE_URLS = ('https://sources.easybuild.io/',) +DEFAULT_EXTRA_SOURCE_URLS = ('https://sources.easybuild.io',) # Filter these CUDA libraries by default from the RPATH sanity check. # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build # against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9d44391877..1f8611d9e8 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -73,8 +73,7 @@ from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS -from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR -from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE +from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN From 65410ff68117d0fe5c49de491f70c2f7ba70ceae Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Wed, 14 Aug 2024 21:18:31 -0400 Subject: [PATCH 07/14] Don't have an import alone on its line. --- easybuild/tools/options.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 1f8611d9e8..bbc8cd9173 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,8 +69,7 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS -from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS, DEFAULT_PR_TARGET_ACCOUNT from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE From 6414ab0f30d56090f7aa07ec7d322f615cda3564 Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Wed, 14 Aug 2024 21:20:50 -0400 Subject: [PATCH 08/14] Make houndbot happy. --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index bbc8cd9173..c17d47fae7 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,9 +69,9 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT -from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS +from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS, DEFAULT_PR_TARGET_ACCOUNT from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS From 6d98a2e815debaca63f3ff6e6bd0a400429ececd Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Thu, 15 Aug 2024 19:54:44 -0400 Subject: [PATCH 09/14] Fix the order of imports. --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index c17d47fae7..dd439ed3c9 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -71,7 +71,7 @@ from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT -from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS From dd86456fe0310eb066c083fb7aa535cf2f3bbdbc Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Sat, 31 Aug 2024 16:56:07 -0400 Subject: [PATCH 10/14] Address review comments. --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/config.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a503318574..527b31e796 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,7 +70,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, EASYBUILD_SOURCES_URL from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index e36e48179a..df27a10f36 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -120,7 +120,8 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' -DEFAULT_EXTRA_SOURCE_URLS = ('https://sources.easybuild.io',) +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' +DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,) # Filter these CUDA libraries by default from the RPATH sanity check. # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build # against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should From d454bade9b8299a4cc405bc467ed5c898defc429 Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Sat, 31 Aug 2024 17:00:58 -0400 Subject: [PATCH 11/14] Chose the way that made houndbot happy. --- easybuild/framework/easyblock.py | 4 ++-- easybuild/tools/config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 527b31e796..a86fb9f690 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,7 +70,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, EASYBUILD_SOURCES_URL +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths @@ -105,7 +105,7 @@ from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION - +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index df27a10f36..f0874771d8 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -48,6 +48,7 @@ from easybuild.base import fancylogger from easybuild.base.frozendict import FrozenDictKnownKeys +from easybuild.framework.easyblock import EASYBUILD_SOURCES_URL from easybuild.tools.build_log import EasyBuildError from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type @@ -120,7 +121,6 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' -EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,) # Filter these CUDA libraries by default from the RPATH sanity check. # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build From 0317785d0f1036b420c68f7646ab4b4a6bba6e57 Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Sat, 31 Aug 2024 18:50:35 -0400 Subject: [PATCH 12/14] Need to revert to EASYBUILD_SOURCES_URL in easyblock, importing it from config in easyblock leads to circular imports. Not sure how to stop houndbot from howling. --- easybuild/framework/easyblock.py | 3 +-- easybuild/tools/config.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a86fb9f690..8e4f0842a3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,7 +70,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, EASYBUILD_SOURCES_URL from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths @@ -105,7 +105,6 @@ from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION -EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index f0874771d8..df27a10f36 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -48,7 +48,6 @@ from easybuild.base import fancylogger from easybuild.base.frozendict import FrozenDictKnownKeys -from easybuild.framework.easyblock import EASYBUILD_SOURCES_URL from easybuild.tools.build_log import EasyBuildError from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type @@ -121,6 +120,7 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,) # Filter these CUDA libraries by default from the RPATH sanity check. # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build From 046e0a6c8c5571c410656fb97a6dbb846358aa88 Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Sat, 31 Aug 2024 21:20:41 -0400 Subject: [PATCH 13/14] Remove unused import, define variable in two locations. --- easybuild/framework/easyblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8e4f0842a3..a86fb9f690 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -70,7 +70,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, EASYBUILD_SOURCES_URL +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths @@ -105,6 +105,7 @@ from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] From f5fcdb437de23e4751b59a4b2cc71a514e83513b Mon Sep 17 00:00:00 2001 From: Joey Dumont Date: Sun, 8 Sep 2024 17:44:30 -0400 Subject: [PATCH 14/14] Don't repeat definitions. --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index a86fb9f690..bd454a5f34 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -72,6 +72,7 @@ from easybuild.tools.build_log import print_error, print_msg, print_warning from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES +from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env @@ -105,7 +106,6 @@ from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION -EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]