diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py new file mode 100644 index 0000000000..982bc41132 --- /dev/null +++ b/easybuild/framework/easystack.py @@ -0,0 +1,234 @@ +# Copyright 2020-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# +""" +Support for easybuild-ing from multiple easyconfigs based on +information obtained from provided file (easystack) with build specifications. + +:author: Denis Kristak (Inuits) +:author: Pavel Grochal (Inuits) +""" + +from easybuild.base import fancylogger +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.utilities import only_if_module_is_available +try: + import yaml +except ImportError: + pass +_log = fancylogger.getLogger('easystack', fname=False) + + +class EasyStack(object): + """One class instance per easystack. General options + list of all SoftwareSpecs instances""" + + def __init__(self): + self.easybuild_version = None + self.robot = False + self.software_list = [] + + def compose_ec_filenames(self): + """Returns a list of all easyconfig names""" + ec_filenames = [] + for sw in self.software_list: + full_ec_version = det_full_ec_version({ + 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, + 'version': sw.version, + 'versionsuffix': sw.versionsuffix, + }) + ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) + ec_filenames.append(ec_filename) + return ec_filenames + + # flags applicable to all sw (i.e. robot) + def get_general_options(self): + """Returns general options (flags applicable to all sw (i.e. --robot))""" + general_options = {} + # TODO add support for general_options + # general_options['robot'] = self.robot + # general_options['easybuild_version'] = self.easybuild_version + return general_options + + +class SoftwareSpecs(object): + """Contains information about every software that should be installed""" + + def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_name): + self.name = name + self.version = version + self.toolchain_version = toolchain_version + self.toolchain_name = toolchain_name + self.versionsuffix = versionsuffix + + +class EasyStackParser(object): + """Parser for easystack files (in YAML syntax).""" + + @only_if_module_is_available('yaml', pkgname='PyYAML') + @staticmethod + def parse(filepath): + """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" + yaml_txt = read_file(filepath) + easystack_raw = yaml.safe_load(yaml_txt) + easystack = EasyStack() + + try: + software = easystack_raw["software"] + except KeyError: + wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found" + raise EasyBuildError(wrong_structure_file) + + # assign software-specific easystack attributes + for name in software: + # ensure we have a string value (YAML parser returns type = dict + # if levels under the current attribute are present) + name = str(name) + try: + toolchains = software[name]['toolchains'] + except KeyError: + raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) + for toolchain in toolchains: + toolchain = str(toolchain) + toolchain_parts = toolchain.split('-', 1) + if len(toolchain_parts) == 2: + toolchain_name, toolchain_version = toolchain_parts + elif len(toolchain_parts) == 1: + toolchain_name, toolchain_version = toolchain, '' + else: + raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", + name, filepath, toolchain_parts) + + try: + # if version string containts asterisk or labels, raise error (asterisks not supported) + versions = toolchains[toolchain]['versions'] + except TypeError as err: + wrong_structure_err = "An error occurred when interpreting " + wrong_structure_err += "the data for software %s: %s" % (name, err) + raise EasyBuildError(wrong_structure_err) + if '*' in str(versions): + asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. " + asterisk_err += "Wildcard feature is not supported yet." + raise EasyBuildError(asterisk_err, name, filepath) + + # yaml versions can be in different formats in yaml file + # firstly, check if versions in yaml file are read as a dictionary. + # Example of yaml structure: + # ======================================================================== + # versions: + # 2.25: + # 2.23: + # versionsuffix: '-R-4.0.0' + # ======================================================================== + if isinstance(versions, dict): + for version in versions: + if versions[version] is not None: + version_spec = versions[version] + if 'versionsuffix' in version_spec: + versionsuffix = str(version_spec['versionsuffix']) + else: + versionsuffix = '' + if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec): + lab_err = "EasyStack specifications of '%s' in %s " + lab_err += "contain labels. Labels aren't supported yet." + raise EasyBuildError(lab_err, name, filepath) + else: + versionsuffix = '' + + specs = { + 'name': name, + 'toolchain_name': toolchain_name, + 'toolchain_version': toolchain_version, + 'version': version, + 'versionsuffix': versionsuffix, + } + sw = SoftwareSpecs(**specs) + + # append newly created class instance to the list in instance of EasyStack class + easystack.software_list.append(sw) + continue + + # is format read as a list of versions? + # ======================================================================== + # versions: + # [2.24, 2.51] + # ======================================================================== + elif isinstance(versions, list): + versions_list = versions + + # format = multiple lines without ':' (read as a string)? + # ======================================================================== + # versions: + # 2.24 + # 2.51 + # ======================================================================== + elif isinstance(versions, str): + versions_list = str(versions).split() + + # format read as float (containing one version only)? + # ======================================================================== + # versions: + # 2.24 + # ======================================================================== + elif isinstance(versions, float): + versions_list = [str(versions)] + + # if no version is a dictionary, versionsuffix isn't specified + versionsuffix = '' + + for version in versions_list: + sw = SoftwareSpecs( + name=name, version=version, versionsuffix=versionsuffix, + toolchain_name=toolchain_name, toolchain_version=toolchain_version) + # append newly created class instance to the list in instance of EasyStack class + easystack.software_list.append(sw) + + # assign general easystack attributes + easystack.easybuild_version = easystack_raw.get('easybuild_version', None) + easystack.robot = easystack_raw.get('robot', False) + + return easystack + + +def parse_easystack(filepath): + """Parses through easystack file, returns what EC are to be installed together with their options.""" + log_msg = "Support for easybuild-ing from multiple easyconfigs based on " + log_msg += "information obtained from provided file (easystack) with build specifications." + _log.experimental(log_msg) + _log.info("Building from easystack: '%s'" % filepath) + + # class instance which contains all info about planned build + easystack = EasyStackParser.parse(filepath) + + easyconfig_names = easystack.compose_ec_filenames() + + general_options = easystack.get_general_options() + + _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: \n'%s'" % "',\n'".join(easyconfig_names)) + if len(general_options) != 0: + _log.debug("General options for installation are: \n%s" % str(general_options)) + else: + _log.debug("No general options were specified in easystack") + + return easyconfig_names, general_options diff --git a/easybuild/main.py b/easybuild/main.py index bd7a0b0f5e..4bcb8ed7d8 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -47,6 +47,7 @@ from easybuild.framework.easyblock import build_and_install_one, inject_checksums from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check @@ -223,6 +224,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) + # if easystack is provided with the command, commands with arguments from it will be executed + if options.easystack: + # TODO add general_options (i.e. robot) to build options + orig_paths, general_options = parse_easystack(options.easystack) + if general_options: + raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.") + # check whether packaging is supported when it's being used if options.package: check_pkg_support() diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 026ae528b1..2e1d827c6a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -604,6 +604,8 @@ def informative_options(self): 'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False), 'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False), 'terse': ("Terse output (machine-readable)", None, 'store_true', False), + 'easystack': ("Path to easystack file in YAML format, specifying details of a software stack", + None, 'store', None), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index f4e6dafe0f..e878489043 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -173,7 +173,7 @@ def wrap(orig): pass if imported is None: - raise ImportError("None of the specified modules %s is available" % ', '.join(modnames)) + raise ImportError("None of the specified modules (%s) is available" % ', '.join(modnames)) else: return orig diff --git a/test/framework/easystacks/test_easystack_asterisk.yaml b/test/framework/easystacks/test_easystack_asterisk.yaml new file mode 100644 index 0000000000..7f440636cd --- /dev/null +++ b/test/framework/easystacks/test_easystack_asterisk.yaml @@ -0,0 +1,6 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + "2.11.*" \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml new file mode 100644 index 0000000000..4272aaf0fc --- /dev/null +++ b/test/framework/easystacks/test_easystack_basic.yaml @@ -0,0 +1,13 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + 2.25: + 2.26: + toy: + toolchains: + gompi-2018a: + versions: + 0.0: + versionsuffix: '-test' \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml new file mode 100644 index 0000000000..51a113523f --- /dev/null +++ b/test/framework/easystacks/test_easystack_labels.yaml @@ -0,0 +1,7 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + 3.11: + exclude-labels: arch:aarch64 diff --git a/test/framework/easystacks/test_easystack_wrong_structure.yaml b/test/framework/easystacks/test_easystack_wrong_structure.yaml new file mode 100644 index 0000000000..a328b5413b --- /dev/null +++ b/test/framework/easystacks/test_easystack_wrong_structure.yaml @@ -0,0 +1,6 @@ +software: + Bioconductor: + toolchains: + # foss-2020a: + versions: + 3.11 \ No newline at end of file diff --git a/test/framework/general.py b/test/framework/general.py index 07e8c36011..fd43213be6 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -88,7 +88,7 @@ def bar(): def bar2(): pass - err_pat = "ImportError: None of the specified modules nosuchmodule, anothernosuchmodule is available" + err_pat = r"ImportError: None of the specified modules \(nosuchmodule, anothernosuchmodule\) is available" self.assertErrorRegex(EasyBuildError, err_pat, bar2) class Foo(): diff --git a/test/framework/options.py b/test/framework/options.py index e2c90a22be..21f24e3f0e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -42,6 +42,7 @@ import easybuild.tools.toolchain from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig @@ -107,11 +108,14 @@ def setUp(self): self.orig_terminal_supports_colors = easybuild.tools.options.terminal_supports_colors self.orig_os_getuid = easybuild.main.os.getuid + self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL def tearDown(self): """Clean up after test.""" easybuild.main.os.getuid = self.orig_os_getuid easybuild.tools.options.terminal_supports_colors = self.orig_terminal_supports_colors + easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental + super(CommandLineOptionsTest, self).tearDown() def purge_environment(self): @@ -5453,6 +5457,69 @@ def test_sysroot(self): os.environ['EASYBUILD_SYSROOT'] = doesnotexist self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) + # end-to-end testing of unknown filename + def test_easystack_wrong_read(self): + """Test for --easystack when wrong name is provided""" + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml') + args = ['--easystack', toy_easystack, '--experimental'] + expected_err = "No such file or directory: '%s'" % toy_easystack + self.assertErrorRegex(EasyBuildError, expected_err, self.eb_main, args, raise_error=True) + + # testing basics - end-to-end + # expecting successful build + def test_easystack_basic(self): + """Test for --easystack -> success case""" + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') + + args = ['--easystack', toy_easystack, '--stop', '--debug', '--experimental'] + stdout, err = self.eb_main(args, do_build=True, return_error=True) + patterns = [ + r"[\S\s]*INFO Building from easystack:[\S\s]*", + r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs:.*?[\n]" + r"[\S\s]*INFO building and installing binutils/2\.25-GCCcore-4\.9\.3[\S\s]*", + r"[\S\s]*INFO building and installing binutils/2\.26-GCCcore-4\.9\.3[\S\s]*", + r"[\S\s]*INFO building and installing toy/0\.0-gompi-2018a-test[\S\s]*", + r"[\S\s]*INFO COMPLETED: Installation STOPPED successfully[\S\s]*", + r"[\S\s]*INFO Build succeeded for 3 out of 3[\S\s]*" + ] + for pattern in patterns: + regex = re.compile(pattern) + self.assertTrue(regex.match(stdout) is not None) + + def test_easystack_wrong_structure(self): + """Test for --easystack when yaml easystack has wrong structure""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') + + expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" + expected_err += r" 'float' object is not subscriptable[\S\s]*" + expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*" + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) + + def test_easystack_asterisk(self): + """Test for --easystack when yaml easystack contains asterisk (wildcard)""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') + + expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " + expected_err += "Wildcard feature is not supported yet." + + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) + + def test_easystack_labels(self): + """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') + + error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " + error_msg += "Labels aren't supported yet." + self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, toy_easystack) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz b/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz new file mode 100644 index 0000000000..98f5271678 Binary files /dev/null and b/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz differ diff --git a/test/framework/sandbox/sources/t/toy/exts-git.tar.gz b/test/framework/sandbox/sources/t/toy/exts-git.tar.gz new file mode 100644 index 0000000000..afac3590b1 Binary files /dev/null and b/test/framework/sandbox/sources/t/toy/exts-git.tar.gz differ