diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f4c4be464e..c0e1810021 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -56,7 +56,6 @@ from easybuild.framework.easyconfig.format.convert import Dependency from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec -from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION, is_yeb_format from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig @@ -571,10 +570,7 @@ def __str__(self): def filename(self): """Determine correct filename for this easyconfig file.""" - if is_yeb_format(self.path, self.rawtxt): - ext = YEB_FORMAT_EXTENSION - else: - ext = EB_FORMAT_EXTENSION + ext = EB_FORMAT_EXTENSION return '%s-%s%s' % (self.name, det_full_ec_version(self), ext) diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py deleted file mode 100644 index acfab3fb22..0000000000 --- a/easybuild/framework/easyconfig/format/yeb.py +++ /dev/null @@ -1,169 +0,0 @@ -# # -# Copyright 2013-2023 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -# # -""" -YAML easyconfig format (.yeb) -Useful: http://www.yaml.org/spec/1.2/spec.html - -Authors: - -* Caroline De Brouwer (Ghent University) -* Kenneth Hoste (Ghent University) -""" -import copy -import os -import platform - -from easybuild.base import fancylogger -from easybuild.framework.easyconfig.format.format import EasyConfigFormat -from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict -from easybuild.tools import LooseVersion -from easybuild.tools.py2vs3 import string_type -from easybuild.tools.utilities import INDENT_4SPACES, only_if_module_is_available, quote_str - - -_log = fancylogger.getLogger('easyconfig.format.yeb', fname=False) - - -YAML_DIR = r'%YAML' -YAML_SEP = '---' -YEB_FORMAT_EXTENSION = '.yeb' -YAML_SPECIAL_CHARS = set(":{}[],&*#?|-<>=!%@\\") - - -def yaml_join(loader, node): - """ - defines custom YAML join function. - see http://stackoverflow.com/questions/5484016/ - how-can-i-do-string-concatenation-or-string-replacement-in-yaml/23212524#23212524 - :param loader: the YAML Loader - :param node: the YAML (sequence) node - """ - seq = loader.construct_sequence(node) - return ''.join([str(i) for i in seq]) - - -try: - import yaml - # register the tag handlers - if LooseVersion(platform.python_version()) < LooseVersion(u'2.7'): - yaml.add_constructor('!join', yaml_join) - else: - yaml.add_constructor(u'!join', yaml_join, Loader=yaml.SafeLoader) -except ImportError: - pass - - -class FormatYeb(EasyConfigFormat): - """Support for easyconfig YAML format""" - USABLE = True - - def __init__(self): - """FormatYeb constructor""" - super(FormatYeb, self).__init__() - self.log.experimental("Parsing .yeb easyconfigs") - - def validate(self): - """Format validation""" - _log.info(".yeb format validation isn't implemented (yet) - validation always passes") - return True - - def get_config_dict(self): - """ - Return parsed easyconfig as a dictionary, based on specified arguments. - """ - # avoid passing anything by reference, so next time get_config_dict is called - # we can be sure we return a dictionary that correctly reflects the contents of the easyconfig file - return copy.deepcopy(self.parsed_yeb) - - @only_if_module_is_available('yaml') - def parse(self, txt): - """ - Process YAML file - """ - txt = self._inject_constants_dict(txt) - if LooseVersion(platform.python_version()) < LooseVersion(u'2.7'): - self.parsed_yeb = yaml.load(txt) - else: - self.parsed_yeb = yaml.load(txt, Loader=yaml.SafeLoader) - - def _inject_constants_dict(self, txt): - """Inject constants so they are resolved when actually parsing the YAML text.""" - constants_dict = build_easyconfig_constants_dict() - - lines = txt.splitlines() - - # extract possible YAML header, for example - # %YAML 1.2 - # --- - yaml_header = [] - for i, line in enumerate(lines): - if line.startswith(YAML_DIR): - if lines[i + 1].startswith(YAML_SEP): - yaml_header.extend([lines.pop(i), lines.pop(i)]) - - injected_constants = ['__CONSTANTS__: '] - for key, value in constants_dict.items(): - injected_constants.append('%s- &%s %s' % (INDENT_4SPACES, key, quote_str(value))) - - full_txt = '\n'.join(yaml_header + injected_constants + lines) - - return full_txt - - def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): - """Dump parsed easyconfig in .yeb format""" - raise NotImplementedError("Dumping of .yeb easyconfigs not supported yet") - - def extract_comments(self, txt): - """Extract comments from easyconfig file""" - self.log.debug("Not extracting comments from .yeb easyconfigs") - - -def is_yeb_format(filename, rawcontent): - """ - Determine whether easyconfig is in .yeb format. - If filename is None, rawcontent will be used to check the format. - """ - isyeb = False - if filename: - isyeb = os.path.splitext(filename)[-1] == YEB_FORMAT_EXTENSION - else: - # if one line like 'name: ' is found, this must be YAML format - for line in rawcontent.splitlines(): - if line.startswith('name: '): - isyeb = True - return isyeb - - -def quote_yaml_special_chars(val): - """ - Single-quote values that contain special characters, specifically to be used in YAML context (.yeb files) - Single quotes inside the string are escaped by doubling them. - (see: http://symfony.com/doc/current/components/yaml/yaml_format.html#strings) - """ - if isinstance(val, string_type): - if "'" in val or YAML_SPECIAL_CHARS.intersection(val): - val = "'%s'" % val.replace("'", "''") - - return val diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index fa01d1b297..a9b084cc12 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -37,7 +37,6 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig.format.format import FORMAT_DEFAULT_VERSION from easybuild.framework.easyconfig.format.format import get_format_version, get_format_version_classes -from easybuild.framework.easyconfig.format.yeb import FormatYeb, is_yeb_format from easybuild.framework.easyconfig.types import PARAMETER_TYPES, check_type_of_param_value from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, write_file @@ -189,11 +188,8 @@ def _get_format_version_class(self): def _set_formatter(self, filename): """Obtain instance of the formatter""" if self._formatter is None: - if is_yeb_format(filename, self.rawcontent): - self._formatter = FormatYeb() - else: - klass = self._get_format_version_class() - self._formatter = klass() + klass = self._get_format_version_class() + self._formatter = klass() self._formatter.parse(self.rawcontent) def set_format_text(self): diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 62968f9de0..48dc1910b0 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -51,7 +51,6 @@ from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, EasyConfig from easybuild.framework.easyconfig.easyconfig import create_paths, det_file_info, get_easyblock_class from easybuild.framework.easyconfig.easyconfig import process_easyconfig -from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning @@ -412,7 +411,7 @@ def parse_easyconfigs(paths, validate=True): return easyconfigs, generated_ecs -def stats_to_str(stats, isyeb=False): +def stats_to_str(stats): """ Pretty print build statistics to string. """ @@ -422,13 +421,7 @@ def stats_to_str(stats, isyeb=False): txt = "{\n" pref = " " for key in sorted(stats): - if isyeb: - val = stats[key] - if isinstance(val, tuple): - val = list(val) - key, val = quote_yaml_special_chars(key), quote_yaml_special_chars(val) - else: - key, val = quote_str(key), quote_str(stats[key]) + key, val = quote_str(key), quote_str(stats[key]) txt += "%s%s: %s,\n" % (pref, key, val) txt += "}" return txt diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 761cdc262f..100fd6a72f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -55,7 +55,6 @@ from easybuild.framework.easyconfig.easyconfig import HAVE_AUTOPEP8 from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict -from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, get_paths_for from easybuild.toolchains.compiler.systemcompiler import TC_CONSTANT_SYSTEM from easybuild.tools import LooseVersion, build_log, run # build_log should always stay there, to ensure EasyBuildLog @@ -1160,11 +1159,11 @@ def _postprocess_config(self): # which makes it susceptible to 'eating' the following argument/option; # for example: with 'eb -r foo', 'foo' must be an existing directory (or 'eb foo -r' should be used); # when multiple directories are specified, we deliberately do not enforce that all of them exist; - # if a single argument is passed to --robot/-r that ends with '.eb' or '.yeb', we assume it's an easyconfig + # if a single argument is passed to --robot/-r that ends with '.eb' we assume it's an easyconfig if len(self.options.robot) == 1: robot_arg = self.options.robot[0] if not os.path.isdir(robot_arg): - if robot_arg.endswith(EB_FORMAT_EXTENSION) or robot_arg.endswith(YEB_FORMAT_EXTENSION): + if robot_arg.endswith(EB_FORMAT_EXTENSION): info_msg = "Sole --robot argument %s is not an existing directory, " info_msg += "promoting it to a stand-alone argument since it looks like an easyconfig file name" self.log.info(info_msg, robot_arg) diff --git a/easybuild/tools/repository/filerepo.py b/easybuild/tools/repository/filerepo.py index c7d0cb366d..a35c7054f0 100644 --- a/easybuild/tools/repository/filerepo.py +++ b/easybuild/tools/repository/filerepo.py @@ -43,7 +43,6 @@ from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION -from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION, is_yeb_format from easybuild.framework.easyconfig.tools import stats_to_str from easybuild.tools.filetools import copy_file, mkdir, read_file, write_file from easybuild.tools.repository.repository import Repository @@ -85,14 +84,8 @@ def add_easyconfig(self, cfg, name, version, stats, previous): # create directory for eb file full_path = os.path.join(self.wc, self.subdir, name) - yeb_format = is_yeb_format(cfg, None) - if yeb_format: - extension = YEB_FORMAT_EXTENSION - prefix = "buildstats: [" - - else: - extension = EB_FORMAT_EXTENSION - prefix = "buildstats = [" + extension = EB_FORMAT_EXTENSION + prefix = "buildstats = [" # destination dest = os.path.join(full_path, "%s-%s%s" % (name, version, extension)) @@ -109,10 +102,10 @@ def add_easyconfig(self, cfg, name, version, stats, previous): if previous: statstxt = statscomment + statsprefix + '\n' for entry in previous + [stats]: - statstxt += stats_to_str(entry, isyeb=yeb_format) + ',\n' + statstxt += stats_to_str(entry) + ',\n' statstxt += statssuffix else: - statstxt = statscomment + statsprefix + stats_to_str(stats, isyeb=yeb_format) + statssuffix + statstxt = statscomment + statsprefix + stats_to_str(stats) + statssuffix txt += statstxt write_file(dest, txt) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 627da42bb6..cafef4ba73 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -211,7 +211,7 @@ 'python-hglib': ('hglib', "using Mercurial repository as easyconfigs archive"), 'requests': (None, "fallback library for downloading files"), 'Rich': (None, "eb command rich terminal output"), - 'PyYAML': ('yaml', "easystack files and .yeb easyconfig format"), + 'PyYAML': ('yaml', "easystack files easyconfig format"), 'setuptools': ('pkg_resources', "obtaining information on Python packages via pkg_resources module"), } diff --git a/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb index b2195b5ce0..a4fba84756 100644 --- a/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb +++ b/test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-intel-2018a.eb @@ -29,8 +29,7 @@ dependencies = [ # it's nice to have an up to date openssl for security reasons ] -# zlib is only included here for the sake of testing parsing of .yeb easyconfigs! -osdependencies = ['zlib', ('openssl-devel', 'libssl-dev', 'libopenssl-devel')] +osdependencies = [('openssl-devel', 'libssl-dev', 'libopenssl-devel')] # order is important! # package versions updated May 28th 2015 diff --git a/test/framework/easyconfigs/yeb/CrayCCE-5.1.29.yeb b/test/framework/easyconfigs/yeb/CrayCCE-5.1.29.yeb deleted file mode 100644 index 7244848a78..0000000000 --- a/test/framework/easyconfigs/yeb/CrayCCE-5.1.29.yeb +++ /dev/null @@ -1,18 +0,0 @@ -easyblock: Toolchain - -name: CrayCCE -version: 5.1.29 - -homepage: (none) -description: Toolchain using Cray compiler wrapper, using PrgEnv-cray module. - -toolchain: system, system - -dependencies: - # also loads cray-libsci - - name: PrgEnv-cray/5.1.29 - external_module: true - - name: fftw/3.3.4.0 - external_module: true - -moduleclass: toolchain diff --git a/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb b/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb deleted file mode 100644 index 527f7bf8a5..0000000000 --- a/test/framework/easyconfigs/yeb/Python-2.7.10-intel-2018a.yeb +++ /dev/null @@ -1,131 +0,0 @@ -_internal_variables_: - - &numpyversion 1.9.2 - - &scipyversion 0.15.1 - -easyblock: ConfigureMake - -name: Python -version: 2.7.10 - -homepage: http://python.org/ -description: | - Python is a programming language that lets you work more quickly and integrate your systems - more effectively. - -toolchain: {name: intel, version: 2018a} -toolchainopts: {pic: True, opt: True, optarch: True} - -source_urls: ['http://www.python.org/ftp/python/%(version)s/'] -sources: [*SOURCE_TGZ] - -# python needs bzip2 to build the bz2 package -# commented out for testing to avoid having to add them all - dependencies are tested in other files -dependencies: [ -# [bzip2, 1.0.6], -# [zlib, 1.2.8], -# [libreadline, '6.3'], -# [ncurses, '5.9'], -# [SQLite, 3.8.10.2], -# [Tk, 8.6.4, -no-X11], -# [OpenSSL, 1.0.1m], # OS dependency should be preferred if the os version is more recent then this version, its -# nice to have an up to date openssl for security reasons -] - -# zlib is only included here for the sake of testing parsing of .yeb easyconfigs! -osdependencies: [zlib, [openssl-devel, libssl-dev, libopenssl-devel]] - -# order is important! -# package versions updated May 28th 2015 -exts_list: [ - [setuptools, '16.0', { - source_urls: ["https://pypi.python.org/packages/source/s/setuptools/"], - }], - [pip, 7.0.1, { - source_urls: ["https://pypi.python.org/packages/source/p/pip/"], - }], - [nose, 1.3.6, { - source_urls: ["https://pypi.python.org/packages/source/n/nose/"], - }], - [numpy, *numpyversion, { - source_urls: [ - [!join ["http://sourceforge.net/projects/numpy/files/NumPy/", *numpyversion], download] - ], - patches: [ - numpy-1.8.0-mkl.patch, # % numpyversion, - ], - }], - [scipy, *scipyversion, { - source_urls: [ - [!join ["http://sourceforge.net/projects/scipy/files/scipy/", *scipyversion], download]], - }], - [blist, 1.3.6, { - source_urls: ["https://pypi.python.org/packages/source/b/blist/"], - }], - [mpi4py, 1.3.1, { - source_urls: ["http://bitbucket.org/mpi4py/mpi4py/downloads/"], - }], - [paycheck, 1.0.2, { - source_urls: ["https://pypi.python.org/packages/source/p/paycheck/"], - }], - [argparse, 1.3.0, { - source_urls: ["https://pypi.python.org/packages/source/a/argparse/"], - }], - [pbr, 1.0.1, { - source_urls: ["https://pypi.python.org/packages/source/p/pbr/"], - }], - [lockfile, 0.10.2, { - source_urls: ["https://pypi.python.org/packages/source/l/lockfile/"], - }], - [Cython, '0.22', { - source_urls: ["http://www.cython.org/release/"], - }], - [six, 1.9.0, { - source_urls: ["https://pypi.python.org/packages/source/s/six/"], - }], - [dateutil, 2.4.2, { - source_tmpl: python-%(name)s-%(version)s.tar.gz, - source_urls: ["https://pypi.python.org/packages/source/p/python-dateutil/"], - }], - [deap, 1.0.2, { - source_tmpl: "%(name)s-%(version)s.post2.tar.gz", - source_urls: ["https://pypi.python.org/packages/source/d/deap/"], - }], - [decorator, 3.4.2, { - source_urls: ["https://pypi.python.org/packages/source/d/decorator/"], - }], - [arff, 2.0.2, { - source_tmpl: liac-%(name)s-%(version)s.zip, - source_urls: ["https://pypi.python.org/packages/source/l/liac-arff/"], - }], - [pycrypto, 2.6.1, { - modulename: Crypto, - source_urls: ["http://ftp.dlitz.net/pub/dlitz/crypto/pycrypto/"], - }], - [ecdsa, '0.13', { - source_urls: ["https://pypi.python.org/packages/source/e/ecdsa/"], - }], - [paramiko, 1.15.2, { - source_urls: ["https://pypi.python.org/packages/source/p/paramiko/"], - }], - [pyparsing, 2.0.3, { - source_urls: ["https://pypi.python.org/packages/source/p/pyparsing/"], - }], - [netifaces, 0.10.4, { - source_urls: ["https://pypi.python.org/packages/source/n/netifaces"], - }], - [netaddr, 0.7.14, { - source_urls: ["https://pypi.python.org/packages/source/n/netaddr"], - }], - [mock, 1.0.1, { - source_urls: ["https://pypi.python.org/packages/source/m/mock"], - }], - [pytz, '2015.4', { - source_urls: ["https://pypi.python.org/packages/source/p/pytz"], - }], - [pandas, 0.16.1, { - source_urls: ["https://pypi.python.org/packages/source/p/pandas"], - }], -] - -moduleclass: lang - diff --git a/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb b/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb deleted file mode 100644 index a9d0afea39..0000000000 --- a/test/framework/easyconfigs/yeb/SQLite-3.8.10.2-foss-2018a.yeb +++ /dev/null @@ -1,32 +0,0 @@ -_internal_variables_: - - &versionstr 3081002 - -easyblock: ConfigureMake - -name: SQLite -version: 3.8.10.2 - -homepage: http://www.sqlite.org/ -# quotes on description to escape special character : -description: "SQLite: SQL Database Engine in a C Library" - -toolchain: foss, 2018a - -# eg. http://www.sqlite.org/2014/sqlite-autoconf-3080600.tar.gz -source_urls: ["http://www.sqlite.org/2015/"] -sources: [!join [sqlite-autoconf-, *versionstr, .tar.gz]] - -# commented out for testing to avoid having to add them all - dependencies are tested in other files -dependencies: [ -# [libreadline, '6.3'], -# [Tcl, 8.6.4], -] - -parallel: 1 - -sanity_check_paths: { - files: [bin/sqlite3, include/sqlite3ext.h, include/sqlite3.h, lib/libsqlite3.a, lib/libsqlite3.so], - dirs: [lib/pkgconfig], -} - -moduleclass: devel diff --git a/test/framework/easyconfigs/yeb/bzip-bad-toolchain.yeb b/test/framework/easyconfigs/yeb/bzip-bad-toolchain.yeb deleted file mode 100644 index f9c0f7f074..0000000000 --- a/test/framework/easyconfigs/yeb/bzip-bad-toolchain.yeb +++ /dev/null @@ -1,26 +0,0 @@ -%YAML 1.2 ---- -# not really (there's an EB_bzip2 easyblock), but fine for use in unit tests -easyblock: ConfigureMake - -name: bzip2 -version: 1.0.6 - -homepage: 'http://www.bzip.org/' -# preserve newlines using '|' to ensure match with description from .eb easyconfig -description: | - bzip2 is a freely available, patent free, high-quality data compressor. It typically - compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical - compressors), whilst being around twice as fast at compression and six times faster at decompression. - -# bad toolchain with four parameters -toolchain: GCC, 4.9, False, 2 -toolchainopts: {pic: True} - -sources: - # SOURCE_TAR_GZ is a known constant, hence the * - - *SOURCE_TAR_GZ -source_urls: - - http://www.bzip.org/%(version)s - -moduleclass: tools diff --git a/test/framework/easyconfigs/yeb/bzip2-1.0.6-GCC-4.9.2.yeb b/test/framework/easyconfigs/yeb/bzip2-1.0.6-GCC-4.9.2.yeb deleted file mode 100644 index a2fd43d756..0000000000 --- a/test/framework/easyconfigs/yeb/bzip2-1.0.6-GCC-4.9.2.yeb +++ /dev/null @@ -1,28 +0,0 @@ -%YAML 1.2 ---- -# not really (there's an EB_bzip2 easyblock), but fine for use in unit tests -easyblock: ConfigureMake - -name: bzip2 -version: 1.0.6 - -homepage: 'http://www.bzip.org/' -# preserve newlines using '|' to ensure match with description from .eb easyconfig -description: | - bzip2 is a freely available, patent free, high-quality data compressor. It typically - compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical - compressors), whilst being around twice as fast at compression and six times faster at decompression. - -toolchain: [GCC, 4.9.2] -toolchainopts: {pic: True} - -sources: - # SOURCE_TAR_GZ is a known constant, hence the * - - *SOURCE_TAR_GZ -source_urls: - - http://www.bzip.org/%(version)s - -builddependencies: - - gzip: 1.6 - -moduleclass: tools diff --git a/test/framework/easyconfigs/yeb/foss-2018a.yeb b/test/framework/easyconfigs/yeb/foss-2018a.yeb deleted file mode 100644 index 99de832ac9..0000000000 --- a/test/framework/easyconfigs/yeb/foss-2018a.yeb +++ /dev/null @@ -1,42 +0,0 @@ -_internal_variables_: - - &version 2018a - - - &comp_name GCC - - &comp_version 6.4.0-2.28 - - &comp [*comp_name, *comp_version] - - - &blaslib OpenBLAS - - &blasver 0.2.20 - - &blas !join [*blaslib, -, *blasver] - - - &comp_mpi_tc [gompi, *version] - - -easyblock: Toolchain - -name: foss -version: *version - -homepage: (none) -description: | - GNU Compiler Collection (GCC) based compiler toolchain, including - OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - -toolchain: {name: system, version: system} - -# compiler toolchain dependencies -# we need GCC and OpenMPI as explicit dependencies instead of gompi toolchain -# because of toolchain preperation functions -dependencies: - - *comp_name: *comp_version - - OpenMPI: 2.1.2 - toolchain: *comp - - *blaslib: *blasver - toolchain: *comp - - FFTW: 3.3.7 - toolchain: *comp_mpi_tc - - ScaLAPACK: 2.0.2 - versionsuffix: !join [-, *blas] - toolchain: *comp_mpi_tc - -moduleclass: toolchain diff --git a/test/framework/easyconfigs/yeb/gzip-1.6-GCC-4.9.2.yeb b/test/framework/easyconfigs/yeb/gzip-1.6-GCC-4.9.2.yeb deleted file mode 100644 index da64d1518d..0000000000 --- a/test/framework/easyconfigs/yeb/gzip-1.6-GCC-4.9.2.yeb +++ /dev/null @@ -1,25 +0,0 @@ -%YAML 1.2 ---- -easyblock: ConfigureMake - -name: gzip -version: 1.6 - -homepage: 'http://www.gnu.org/software/gzip/' -description: - gzip (GNU zip) is a popular data compression program - as a replacement for compress - -toolchain: {name: GCC, version: 4.9.2} - -# http://ftp.gnu.org/gnu/gzip/gzip-1.6.tar.gz -source_urls: [*GNU_SOURCE] -sources: ['%(name)s-%(version)s.tar.gz'] - -# make sure the gzip, gunzip and compress binaries are available after installation -sanity_check_paths: { - files: [bin/gunzip, bin/gzip, bin/uncompress], - dirs: [], -} - -moduleclass: tools diff --git a/test/framework/easyconfigs/yeb/intel-2018a.yeb b/test/framework/easyconfigs/yeb/intel-2018a.yeb deleted file mode 100644 index c1ec0daf29..0000000000 --- a/test/framework/easyconfigs/yeb/intel-2018a.yeb +++ /dev/null @@ -1,24 +0,0 @@ -_internal_variables_: - - &compver 2018.1.163 - -easyblock: Toolchain - -name: intel -version: 2018a - -homepage: http://software.intel.com/en-us/intel-cluster-toolkit-compiler/ -description: - Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and Fortran compilers, - Intel MPI & Intel MKL. - -toolchain: {name: system, version: system} - -# fake intel toolchain easyconfig, no dependencies (good enough for testing) -dependencies: [ - # [icc, *compver], - # [ifort, *compver], - # [impi, 2018.1.163], - # [imkl, 2018.1.163] -] - -moduleclass: toolchain diff --git a/test/framework/easyconfigs/yeb/toy-0.0.yeb b/test/framework/easyconfigs/yeb/toy-0.0.yeb deleted file mode 100644 index 2f5265ea6d..0000000000 --- a/test/framework/easyconfigs/yeb/toy-0.0.yeb +++ /dev/null @@ -1,38 +0,0 @@ -%YAML 1.2 ---- - -name: toy -version: 0.0 - -homepage: 'https://easybuilders.github.io/easybuild' -description: "Toy C program, 100% toy." - -toolchain: system, system - -sources: - - *SOURCE_TAR_GZ - -checksums: [[ - 'be662daa971a640e40be5c804d9d7d10', # default [MD5] - '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ['adler32', '0x998410035'], - ['crc32', '0x1553842328'], - ['md5', 'be662daa971a640e40be5c804d9d7d10'], - ['sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'], - ['size', 273], -]] - -patches: [ - 'toy-0.0_fix-silly-typo-in-printf-statement.patch', - ['toy-extra.txt', 'toy-0.0'], -] - -sanity_check_paths: { - files: [['bin/yot', 'bin/toy']], - dirs: ['bin'], -} - -postinstallcmds: ["echo TOY > %(installdir)s/README"] - -moduleclass: tools -# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/repository.py b/test/framework/repository.py index aa1cf402b7..97d3ead259 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -35,7 +35,6 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from unittest import TextTestRunner -import easybuild.tools.build_log from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file @@ -185,23 +184,6 @@ def check_ec(path, expected_buildstats): path = repo.add_easyconfig(toy_eb_file, 'test', '1.0', {'time': 1.23, 'size': 123}, [{'time': 0.9, 'size': 2}]) check_ec(path, [{'time': 0.9, 'size': 2}, {'time': 1.23, 'size': 123}]) - orig_experimental = easybuild.tools.build_log.EXPERIMENTAL - easybuild.tools.build_log.EXPERIMENTAL = True - - if 'yaml' in sys.modules: - toy_yeb_file = os.path.join(test_easyconfigs, 'yeb', 'toy-0.0.yeb') - path = repo.add_easyconfig(toy_yeb_file, 'test', '1.0', {'time': 1.23}, None) - check_ec(path, [{'time': 1.23}]) - - stats1 = {'time': 1.23, 'size': 123} - stats2 = [{'time': 0.9, 'size': 2}] - path = repo.add_easyconfig(toy_yeb_file, 'test', '1.0', stats1, stats2) - check_ec(path, stats2 + [stats1]) - - easybuild.tools.build_log.EXPERIMENTAL = orig_experimental - else: - print("Skipping .yeb part of test_add_easyconfig (no PyYAML available)") - def tearDown(self): """Clean up after test.""" super(RepositoryTest, self).tearDown() diff --git a/test/framework/suite.py b/test/framework/suite.py index 6a40e35754..e109848340 100755 --- a/test/framework/suite.py +++ b/test/framework/suite.py @@ -81,7 +81,6 @@ import test.framework.tweak as tw import test.framework.utilities_test as u import test.framework.variables as v -import test.framework.yeb as y # set plain text key ring to be used, # so a GitHub token stored in it can be obtained without having to provide a password @@ -121,7 +120,7 @@ # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, d, bl, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, lic, f_c, - tw, p, i, pkg, env, et, y, st, h, ct, lib, u, es, ou] + tw, p, i, pkg, env, et, st, h, ct, lib, u, es, ou] SUITE = unittest.TestSuite([x.suite() for x in tests]) res = unittest.TextTestRunner().run(SUITE) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9d4fdf5de5..be1421ec85 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2562,20 +2562,15 @@ def test_toy_build_enhanced_sanity_check(self): del sys.modules['easybuild.easyblocks.toy'] def test_toy_dumped_easyconfig(self): - """ Test dumping of file in eb_filerepo in both .eb and .yeb format """ + """ Test dumping of file in eb_filerepo in both .eb format """ filename = 'toy-0.0' test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') paths = [ os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', '%s.eb' % filename), - os.path.join(test_ecs_dir, 'yeb', '%s.yeb' % filename), ] for path in paths: - if path.endswith('.yeb') and 'yaml' not in sys.modules: - print("Skipping .yeb part of test_toy_dumped_easyconfig (no PyYAML available)") - continue - args = [ path, '--experimental', diff --git a/test/framework/yeb.py b/test/framework/yeb.py deleted file mode 100644 index 74a2a47875..0000000000 --- a/test/framework/yeb.py +++ /dev/null @@ -1,214 +0,0 @@ -# # -# Copyright 2015-2023 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -# # -""" -Unit tests for .yeb easyconfig format - -@author: Caroline De Brouwer (Ghent University) -@author: Kenneth Hoste (Ghent University) -""" -import glob -import os -import platform -import sys -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config -from unittest import TextTestRunner - -import easybuild.tools.build_log -from easybuild.framework.easyconfig.easyconfig import EasyConfig -from easybuild.framework.easyconfig.format.yeb import is_yeb_format -from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import module_classes -from easybuild.tools.filetools import read_file - - -try: - import yaml -except ImportError: - pass - - -class YebTest(EnhancedTestCase): - """ Testcase for run module """ - - def setUp(self): - """Test setup.""" - super(YebTest, self).setUp() - self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL - easybuild.tools.build_log.EXPERIMENTAL = True - - def tearDown(self): - """Test cleanup.""" - super(YebTest, self).tearDown() - easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental - - def test_parse_yeb(self): - """Test parsing of .yeb easyconfigs.""" - if 'yaml' not in sys.modules: - print("Skipping test_parse_yeb (no PyYAML available)") - return - - build_options = { - 'check_osdeps': False, - 'external_modules_metadata': {}, - 'silent': True, - 'valid_module_classes': module_classes(), - } - init_config(build_options=build_options) - easybuild.tools.build_log.EXPERIMENTAL = True - - testdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs = os.path.join(testdir, 'easyconfigs') - test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') - - # test parsing - test_files = [ - 'bzip2-1.0.6-GCC-4.9.2', - 'gzip-1.6-GCC-4.9.2', - 'foss-2018a', - 'intel-2018a', - 'SQLite-3.8.10.2-foss-2018a', - 'Python-2.7.10-intel-2018a', - 'CrayCCE-5.1.29', - 'toy-0.0', - ] - - for filename in test_files: - ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename)) - # compare with parsed result of .eb easyconfig - ec_file = glob.glob(os.path.join(test_easyconfigs, 'test_ecs', '*', '*', '%s.eb' % filename))[0] - ec_eb = EasyConfig(ec_file) - - for key in sorted(ec_yeb.asdict()): - eb_val = ec_eb[key] - yeb_val = ec_yeb[key] - if key == 'description': - # multi-line string is always terminated with '\n' in YAML, so strip it off - yeb_val = yeb_val.strip() - - self.assertEqual(yeb_val, eb_val) - - def test_yeb_get_config_obj(self): - """Test get_config_dict method.""" - testdir = os.path.dirname(os.path.abspath(__file__)) - test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') - ec = EasyConfig(os.path.join(test_yeb_easyconfigs, 'toy-0.0.yeb')) - ecdict = ec.parser.get_config_dict() - - # changes to this dict should not affect the return value of the next call to get_config_dict - fn = 'test.tar.gz' - ecdict['sources'].append(fn) - - ecdict_bis = ec.parser.get_config_dict() - self.assertIn(fn, ecdict['sources']) - self.assertNotIn(fn, ecdict_bis['sources']) - - def test_is_yeb_format(self): - """ Test is_yeb_format function """ - testdir = os.path.dirname(os.path.abspath(__file__)) - test_yeb = os.path.join(testdir, 'easyconfigs', 'yeb', 'bzip2-1.0.6-GCC-4.9.2.yeb') - raw_yeb = read_file(test_yeb) - - self.assertTrue(is_yeb_format(test_yeb, None)) - self.assertTrue(is_yeb_format(None, raw_yeb)) - - test_eb = os.path.join(testdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb') - raw_eb = read_file(test_eb) - - self.assertFalse(is_yeb_format(test_eb, None)) - self.assertFalse(is_yeb_format(None, raw_eb)) - - def test_join(self): - """ Test yaml_join function """ - # skip test if yaml module was not loaded - if 'yaml' not in sys.modules: - print("Skipping test_join (no PyYAML available)") - return - - stream = [ - "variables:", - " - &f foo", - " - &b bar", - "", - "fb1: !join [foo, bar]", - "fb2: !join [*f, bar]", - "fb3: !join [*f, *b]", - ] - - # import here for testing yaml_join separately - from easybuild.framework.easyconfig.format.yeb import yaml_join # noqa - if LooseVersion(platform.python_version()) < LooseVersion(u'2.7'): - loaded = yaml.load('\n'.join(stream)) - else: - loaded = yaml.load(u'\n'.join(stream), Loader=yaml.SafeLoader) - for key in ['fb1', 'fb2', 'fb3']: - self.assertEqual(loaded.get(key), 'foobar') - - def test_bad_toolchain_format(self): - """ Test alternate toolchain format name,version """ - if 'yaml' not in sys.modules: - print("Skipping test_parse_yeb (no PyYAML available)") - return - - # only test bad cases - the right ones are tested with the test files in test_parse_yeb - testdir = os.path.dirname(os.path.abspath(__file__)) - test_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') - expected = r'Can not convert list .* to toolchain dict. Expected 2 or 3 elements' - self.assertErrorRegex(EasyBuildError, expected, EasyConfig, - os.path.join(test_easyconfigs, 'bzip-bad-toolchain.yeb')) - - def test_external_module_toolchain(self): - """Test specifying external (build) dependencies in yaml format.""" - if 'yaml' not in sys.modules: - print("Skipping test_external_module_toolchain (no PyYAML available)") - return - - ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb', 'CrayCCE-5.1.29.yeb') - metadata = { - 'name': ['foo', 'bar'], - 'version': ['1.2.3', '3.2.1'], - 'prefix': '/foo/bar', - } - build_options = { - 'external_modules_metadata': {'fftw/3.3.4.0': metadata}, - 'valid_module_classes': module_classes(), - } - init_config(build_options=build_options) - easybuild.tools.build_log.EXPERIMENTAL = True - - ec = EasyConfig(ecpath) - - self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0') - self.assertEqual(ec.dependencies()[1]['external_module_metadata'], metadata) - - -def suite(): - """ returns all the testcases in this module """ - return TestLoaderFiltered().loadTestsFromTestCase(YebTest, sys.argv[1:]) - - -if __name__ == '__main__': - res = TextTestRunner(verbosity=1).run(suite()) - sys.exit(len(res.failures))