diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 90248fb0cd..c3660f7cbf 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -97,11 +97,16 @@ def what_str_list_tuple(name): """Given name, return separator, class and helptext wrt separator. (Currently supports strlist, strtuple, pathlist, pathtuple) """ - sep = ',' - helpsep = 'comma' if name.startswith('path'): sep = os.pathsep helpsep = 'pathsep' + elif name.startswith('url'): + # | is one of the only characters not in the grammar for URIs (RFC3986) + sep = '|' + helpsep = '|' + else: + sep = ',' + helpsep = 'comma' klass = None if name.endswith('list'): @@ -182,6 +187,7 @@ class ExtOption(CompleterOption): - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings - the path separator is OS-dependent + - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings """ EXTEND_SEPARATOR = ',' @@ -198,7 +204,7 @@ class ExtOption(CompleterOption): TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS - TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']] + TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']] TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST} TYPE_CHECKER.update(Option.TYPE_CHECKER) TYPES = tuple(TYPE_STRLIST + list(Option.TYPES)) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index ce436f1841..bd454a5f34 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -72,6 +72,7 @@ from easybuild.tools.build_log import print_error, print_msg, print_warning from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES +from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env @@ -105,9 +106,6 @@ from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION - -EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' - DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] @@ -897,8 +895,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - # add https://sources.easybuild.io as fallback source URL - source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location)) + # Add additional URLs as configured. + for url in build_option("extra_source_urls"): + url += "/" + name_letter + "/" + location + source_urls.append(url) mkdir(targetdir, parents=True) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6bec64764c..df27a10f36 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -120,6 +120,8 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' +DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,) # Filter these CUDA libraries by default from the RPATH sanity check. # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build # against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should @@ -391,6 +393,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'defaultopt': [ 'default_opt_level', ], + DEFAULT_EXTRA_SOURCE_URLS: [ + 'extra_source_urls', + ], DEFAULT_ALLOW_LOADED_MODULES: [ 'allow_loaded_modules', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index df10ec859e..dd439ed3c9 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,9 +69,9 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT -from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS +from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS @@ -407,6 +407,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig", + "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[|URL]'}), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "