diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py
new file mode 100644
index 0000000000..982bc41132
--- /dev/null
+++ b/easybuild/framework/easystack.py
@@ -0,0 +1,234 @@
+# Copyright 2020-2020 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+#
+"""
+Support for easybuild-ing from multiple easyconfigs based on
+information obtained from provided file (easystack) with build specifications.
+
+:author: Denis Kristak (Inuits)
+:author: Pavel Grochal (Inuits)
+"""
+
+from easybuild.base import fancylogger
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import read_file
+from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
+from easybuild.tools.utilities import only_if_module_is_available
+try:
+ import yaml
+except ImportError:
+ pass
+_log = fancylogger.getLogger('easystack', fname=False)
+
+
+class EasyStack(object):
+ """One class instance per easystack. General options + list of all SoftwareSpecs instances"""
+
+ def __init__(self):
+ self.easybuild_version = None
+ self.robot = False
+ self.software_list = []
+
+ def compose_ec_filenames(self):
+ """Returns a list of all easyconfig names"""
+ ec_filenames = []
+ for sw in self.software_list:
+ full_ec_version = det_full_ec_version({
+ 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version},
+ 'version': sw.version,
+ 'versionsuffix': sw.versionsuffix,
+ })
+ ec_filename = '%s-%s.eb' % (sw.name, full_ec_version)
+ ec_filenames.append(ec_filename)
+ return ec_filenames
+
+ # flags applicable to all sw (i.e. robot)
+ def get_general_options(self):
+ """Returns general options (flags applicable to all sw (i.e. --robot))"""
+ general_options = {}
+ # TODO add support for general_options
+ # general_options['robot'] = self.robot
+ # general_options['easybuild_version'] = self.easybuild_version
+ return general_options
+
+
+class SoftwareSpecs(object):
+ """Contains information about every software that should be installed"""
+
+ def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_name):
+ self.name = name
+ self.version = version
+ self.toolchain_version = toolchain_version
+ self.toolchain_name = toolchain_name
+ self.versionsuffix = versionsuffix
+
+
+class EasyStackParser(object):
+ """Parser for easystack files (in YAML syntax)."""
+
+ @only_if_module_is_available('yaml', pkgname='PyYAML')
+ @staticmethod
+ def parse(filepath):
+ """Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
+ yaml_txt = read_file(filepath)
+ easystack_raw = yaml.safe_load(yaml_txt)
+ easystack = EasyStack()
+
+ try:
+ software = easystack_raw["software"]
+ except KeyError:
+ wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found"
+ raise EasyBuildError(wrong_structure_file)
+
+ # assign software-specific easystack attributes
+ for name in software:
+ # ensure we have a string value (YAML parser returns type = dict
+ # if levels under the current attribute are present)
+ name = str(name)
+ try:
+ toolchains = software[name]['toolchains']
+ except KeyError:
+ raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath)
+ for toolchain in toolchains:
+ toolchain = str(toolchain)
+ toolchain_parts = toolchain.split('-', 1)
+ if len(toolchain_parts) == 2:
+ toolchain_name, toolchain_version = toolchain_parts
+ elif len(toolchain_parts) == 1:
+ toolchain_name, toolchain_version = toolchain, ''
+ else:
+ raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s",
+ name, filepath, toolchain_parts)
+
+ try:
+ # if version string containts asterisk or labels, raise error (asterisks not supported)
+ versions = toolchains[toolchain]['versions']
+ except TypeError as err:
+ wrong_structure_err = "An error occurred when interpreting "
+ wrong_structure_err += "the data for software %s: %s" % (name, err)
+ raise EasyBuildError(wrong_structure_err)
+ if '*' in str(versions):
+ asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. "
+ asterisk_err += "Wildcard feature is not supported yet."
+ raise EasyBuildError(asterisk_err, name, filepath)
+
+ # yaml versions can be in different formats in yaml file
+ # firstly, check if versions in yaml file are read as a dictionary.
+ # Example of yaml structure:
+ # ========================================================================
+ # versions:
+ # 2.25:
+ # 2.23:
+ # versionsuffix: '-R-4.0.0'
+ # ========================================================================
+ if isinstance(versions, dict):
+ for version in versions:
+ if versions[version] is not None:
+ version_spec = versions[version]
+ if 'versionsuffix' in version_spec:
+ versionsuffix = str(version_spec['versionsuffix'])
+ else:
+ versionsuffix = ''
+ if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec):
+ lab_err = "EasyStack specifications of '%s' in %s "
+ lab_err += "contain labels. Labels aren't supported yet."
+ raise EasyBuildError(lab_err, name, filepath)
+ else:
+ versionsuffix = ''
+
+ specs = {
+ 'name': name,
+ 'toolchain_name': toolchain_name,
+ 'toolchain_version': toolchain_version,
+ 'version': version,
+ 'versionsuffix': versionsuffix,
+ }
+ sw = SoftwareSpecs(**specs)
+
+ # append newly created class instance to the list in instance of EasyStack class
+ easystack.software_list.append(sw)
+ continue
+
+ # is format read as a list of versions?
+ # ========================================================================
+ # versions:
+ # [2.24, 2.51]
+ # ========================================================================
+ elif isinstance(versions, list):
+ versions_list = versions
+
+ # format = multiple lines without ':' (read as a string)?
+ # ========================================================================
+ # versions:
+ # 2.24
+ # 2.51
+ # ========================================================================
+ elif isinstance(versions, str):
+ versions_list = str(versions).split()
+
+ # format read as float (containing one version only)?
+ # ========================================================================
+ # versions:
+ # 2.24
+ # ========================================================================
+ elif isinstance(versions, float):
+ versions_list = [str(versions)]
+
+ # if no version is a dictionary, versionsuffix isn't specified
+ versionsuffix = ''
+
+ for version in versions_list:
+ sw = SoftwareSpecs(
+ name=name, version=version, versionsuffix=versionsuffix,
+ toolchain_name=toolchain_name, toolchain_version=toolchain_version)
+ # append newly created class instance to the list in instance of EasyStack class
+ easystack.software_list.append(sw)
+
+ # assign general easystack attributes
+ easystack.easybuild_version = easystack_raw.get('easybuild_version', None)
+ easystack.robot = easystack_raw.get('robot', False)
+
+ return easystack
+
+
+def parse_easystack(filepath):
+ """Parses through easystack file, returns what EC are to be installed together with their options."""
+ log_msg = "Support for easybuild-ing from multiple easyconfigs based on "
+ log_msg += "information obtained from provided file (easystack) with build specifications."
+ _log.experimental(log_msg)
+ _log.info("Building from easystack: '%s'" % filepath)
+
+ # class instance which contains all info about planned build
+ easystack = EasyStackParser.parse(filepath)
+
+ easyconfig_names = easystack.compose_ec_filenames()
+
+ general_options = easystack.get_general_options()
+
+ _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: \n'%s'" % "',\n'".join(easyconfig_names))
+ if len(general_options) != 0:
+ _log.debug("General options for installation are: \n%s" % str(general_options))
+ else:
+ _log.debug("No general options were specified in easystack")
+
+ return easyconfig_names, general_options
diff --git a/easybuild/main.py b/easybuild/main.py
index bd7a0b0f5e..4bcb8ed7d8 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -47,6 +47,7 @@
from easybuild.framework.easyblock import build_and_install_one, inject_checksums
from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
+from easybuild.framework.easystack import parse_easystack
from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs
from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename
from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
@@ -223,6 +224,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
+ # if easystack is provided with the command, commands with arguments from it will be executed
+ if options.easystack:
+ # TODO add general_options (i.e. robot) to build options
+ orig_paths, general_options = parse_easystack(options.easystack)
+ if general_options:
+ raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.")
+
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 026ae528b1..2e1d827c6a 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -604,6 +604,8 @@ def informative_options(self):
'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False),
'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False),
'terse': ("Terse output (machine-readable)", None, 'store_true', False),
+ 'easystack': ("Path to easystack file in YAML format, specifying details of a software stack",
+ None, 'store', None),
})
self.log.debug("informative_options: descr %s opts %s" % (descr, opts))
diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py
index f4e6dafe0f..e878489043 100644
--- a/easybuild/tools/utilities.py
+++ b/easybuild/tools/utilities.py
@@ -173,7 +173,7 @@ def wrap(orig):
pass
if imported is None:
- raise ImportError("None of the specified modules %s is available" % ', '.join(modnames))
+ raise ImportError("None of the specified modules (%s) is available" % ', '.join(modnames))
else:
return orig
diff --git a/test/framework/easystacks/test_easystack_asterisk.yaml b/test/framework/easystacks/test_easystack_asterisk.yaml
new file mode 100644
index 0000000000..7f440636cd
--- /dev/null
+++ b/test/framework/easystacks/test_easystack_asterisk.yaml
@@ -0,0 +1,6 @@
+software:
+ binutils:
+ toolchains:
+ GCCcore-4.9.3:
+ versions:
+ "2.11.*"
\ No newline at end of file
diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml
new file mode 100644
index 0000000000..4272aaf0fc
--- /dev/null
+++ b/test/framework/easystacks/test_easystack_basic.yaml
@@ -0,0 +1,13 @@
+software:
+ binutils:
+ toolchains:
+ GCCcore-4.9.3:
+ versions:
+ 2.25:
+ 2.26:
+ toy:
+ toolchains:
+ gompi-2018a:
+ versions:
+ 0.0:
+ versionsuffix: '-test'
\ No newline at end of file
diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml
new file mode 100644
index 0000000000..51a113523f
--- /dev/null
+++ b/test/framework/easystacks/test_easystack_labels.yaml
@@ -0,0 +1,7 @@
+software:
+ binutils:
+ toolchains:
+ GCCcore-4.9.3:
+ versions:
+ 3.11:
+ exclude-labels: arch:aarch64
diff --git a/test/framework/easystacks/test_easystack_wrong_structure.yaml b/test/framework/easystacks/test_easystack_wrong_structure.yaml
new file mode 100644
index 0000000000..a328b5413b
--- /dev/null
+++ b/test/framework/easystacks/test_easystack_wrong_structure.yaml
@@ -0,0 +1,6 @@
+software:
+ Bioconductor:
+ toolchains:
+ # foss-2020a:
+ versions:
+ 3.11
\ No newline at end of file
diff --git a/test/framework/general.py b/test/framework/general.py
index 07e8c36011..fd43213be6 100644
--- a/test/framework/general.py
+++ b/test/framework/general.py
@@ -88,7 +88,7 @@ def bar():
def bar2():
pass
- err_pat = "ImportError: None of the specified modules nosuchmodule, anothernosuchmodule is available"
+ err_pat = r"ImportError: None of the specified modules \(nosuchmodule, anothernosuchmodule\) is available"
self.assertErrorRegex(EasyBuildError, err_pat, bar2)
class Foo():
diff --git a/test/framework/options.py b/test/framework/options.py
index e2c90a22be..21f24e3f0e 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -42,6 +42,7 @@
import easybuild.tools.toolchain
from easybuild.base import fancylogger
from easybuild.framework.easyblock import EasyBlock
+from easybuild.framework.easystack import parse_easystack
from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE
from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN
from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig
@@ -107,11 +108,14 @@ def setUp(self):
self.orig_terminal_supports_colors = easybuild.tools.options.terminal_supports_colors
self.orig_os_getuid = easybuild.main.os.getuid
+ self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
def tearDown(self):
"""Clean up after test."""
easybuild.main.os.getuid = self.orig_os_getuid
easybuild.tools.options.terminal_supports_colors = self.orig_terminal_supports_colors
+ easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental
+
super(CommandLineOptionsTest, self).tearDown()
def purge_environment(self):
@@ -5453,6 +5457,69 @@ def test_sysroot(self):
os.environ['EASYBUILD_SYSROOT'] = doesnotexist
self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True)
+ # end-to-end testing of unknown filename
+ def test_easystack_wrong_read(self):
+ """Test for --easystack when wrong name is provided"""
+ topdir = os.path.dirname(os.path.abspath(__file__))
+ toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml')
+ args = ['--easystack', toy_easystack, '--experimental']
+ expected_err = "No such file or directory: '%s'" % toy_easystack
+ self.assertErrorRegex(EasyBuildError, expected_err, self.eb_main, args, raise_error=True)
+
+ # testing basics - end-to-end
+ # expecting successful build
+ def test_easystack_basic(self):
+ """Test for --easystack -> success case"""
+ topdir = os.path.dirname(os.path.abspath(__file__))
+ toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml')
+
+ args = ['--easystack', toy_easystack, '--stop', '--debug', '--experimental']
+ stdout, err = self.eb_main(args, do_build=True, return_error=True)
+ patterns = [
+ r"[\S\s]*INFO Building from easystack:[\S\s]*",
+ r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs:.*?[\n]"
+ r"[\S\s]*INFO building and installing binutils/2\.25-GCCcore-4\.9\.3[\S\s]*",
+ r"[\S\s]*INFO building and installing binutils/2\.26-GCCcore-4\.9\.3[\S\s]*",
+ r"[\S\s]*INFO building and installing toy/0\.0-gompi-2018a-test[\S\s]*",
+ r"[\S\s]*INFO COMPLETED: Installation STOPPED successfully[\S\s]*",
+ r"[\S\s]*INFO Build succeeded for 3 out of 3[\S\s]*"
+ ]
+ for pattern in patterns:
+ regex = re.compile(pattern)
+ self.assertTrue(regex.match(stdout) is not None)
+
+ def test_easystack_wrong_structure(self):
+ """Test for --easystack when yaml easystack has wrong structure"""
+ easybuild.tools.build_log.EXPERIMENTAL = True
+ topdir = os.path.dirname(os.path.abspath(__file__))
+ toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml')
+
+ expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:"
+ expected_err += r" 'float' object is not subscriptable[\S\s]*"
+ expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*"
+ self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack)
+
+ def test_easystack_asterisk(self):
+ """Test for --easystack when yaml easystack contains asterisk (wildcard)"""
+ easybuild.tools.build_log.EXPERIMENTAL = True
+ topdir = os.path.dirname(os.path.abspath(__file__))
+ toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml')
+
+ expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. "
+ expected_err += "Wildcard feature is not supported yet."
+
+ self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack)
+
+ def test_easystack_labels(self):
+ """Test for --easystack when yaml easystack contains exclude-labels / include-labels"""
+ easybuild.tools.build_log.EXPERIMENTAL = True
+ topdir = os.path.dirname(os.path.abspath(__file__))
+ toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml')
+
+ error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. "
+ error_msg += "Labels aren't supported yet."
+ self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, toy_easystack)
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz b/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz
new file mode 100644
index 0000000000..98f5271678
Binary files /dev/null and b/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz differ
diff --git a/test/framework/sandbox/sources/t/toy/exts-git.tar.gz b/test/framework/sandbox/sources/t/toy/exts-git.tar.gz
new file mode 100644
index 0000000000..afac3590b1
Binary files /dev/null and b/test/framework/sandbox/sources/t/toy/exts-git.tar.gz differ