diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 860eac3a21..c3bff9a0b7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -69,6 +69,7 @@ from easybuild.tools.build_log import print_error, print_msg, print_warning from easybuild.tools.config import DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES +from easybuild.tools.config import EXTRA_SOURCE_URLS_PRIMARY from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env @@ -103,8 +104,6 @@ from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION -EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' - DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] @@ -821,8 +820,14 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - # add https://sources.easybuild.io as fallback source URL - source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location)) + # add extra-source-urls CLI as either a first check, or a fallback. + for url in build_option("extra_source_urls"): + index = ( + 0 + if build_option("extra_source_urls_priority") == EXTRA_SOURCE_URLS_PRIMARY + else len(source_urls) + ) + source_urls.insert(index, url + "/" + os.path.join(name_letter, location)) mkdir(targetdir, parents=True) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 96daf4c25c..c560b3e2f1 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -113,9 +113,15 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' +DEFAULT_EXTRA_SOURCE_URLS = ('https://sources.easybuild.io',) DEFAULT_WAIT_ON_LOCK_INTERVAL = 60 DEFAULT_WAIT_ON_LOCK_LIMIT = 0 +EXTRA_SOURCE_URLS_PRIMARY = 'primary' +EXTRA_SOURCE_URLS_BACKUP = 'backup' +EXTRA_SOURCE_URLS_PRIORITY_CHOICES = [EXTRA_SOURCE_URLS_PRIMARY, EXTRA_SOURCE_URLS_BACKUP] +DEFAULT_EXTRA_SOURCE_URLS_PRIORITY = EXTRA_SOURCE_URLS_BACKUP + EBROOT_ENV_VAR_ACTIONS = [ERROR, IGNORE, UNSET, WARN] LOADED_MODULES_ACTIONS = [ERROR, IGNORE, PURGE, UNLOAD, WARN] DEFAULT_ALLOW_LOADED_MODULES = ('EasyBuild',) @@ -191,6 +197,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'download_timeout', 'dump_test_report', 'easyblock', + 'extra_source_urls_priority', 'envvars_user_modules', 'extra_modules', 'filter_deps', @@ -357,6 +364,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'defaultopt': [ 'default_opt_level', ], + DEFAULT_EXTRA_SOURCE_URLS: [ + 'extra_source_urls', + ], DEFAULT_ALLOW_LOADED_MODULES: [ 'allow_loaded_modules', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6128c3e650..620551056d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -65,9 +65,11 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS +from easybuild.tools.config import DEFAULT_EXTRA_SOURCE_URLS_PRIORITY, DEFAULT_PR_TARGET_ACCOUNT from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT -from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE +from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, EXTRA_SOURCE_URLS_PRIORITY_CHOICES +from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN @@ -394,6 +396,11 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + "extra-source-urls": ("Specify different URLs to fetch sources from", "strlist", "store", + DEFAULT_EXTRA_SOURCE_URLS), + "extra-source-urls-priority": ("Specify whether sources-url should be tried before or after the " + "ones defined in the easyconfigs", "choice", "store_or_None", + DEFAULT_EXTRA_SOURCE_URLS_PRIORITY, EXTRA_SOURCE_URLS_PRIORITY_CHOICES), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "