From ed756c80bac8f53693c1a1c27fd097889def2c1a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 19:03:40 +0100 Subject: [PATCH 01/14] clean up README --- README.rst | 82 ++++++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 45 deletions(-) diff --git a/README.rst b/README.rst index 2cd7efc455..f071e1049f 100644 --- a/README.rst +++ b/README.rst @@ -1,54 +1,46 @@ -Build status - *master branch (Python 2.4, Python 2.6, Python 2.7)* - -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/ - -Build status - *develop branch (Python 2.4, Python 2.6, Python 2.7)* - -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/ -.. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/badge/icon - :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/ - EasyBuild: building software with ease -------------------------------------- -The easybuild-framework package is the basis for EasyBuild -(http://hpcugent.github.com/easybuild), a software build and -installation framework written in Python that allows you to install -software in a structured, repeatable and robust way. +.. image:: http://hpcugent.github.io/easybuild/images/easybuild_logo_small.png + :align: center + +`EasyBuild `_ is a software build +and installation framework that allows you to manage (scientific) software +on High Performance Computing (HPC) systems in an efficient way. -This package contains the EasyBuild framework that supports the -implementation and use of so-called easyblocks, that implement the -software install procedure for a particular (group of) software +The *easybuild-framework* package is the core of EasyBuild. It +supports the implementation and use of so-called easyblocks which +implement the software install procedure for a particular (group of) software package(s). -The code of the easybuild-framework package is hosted on GitHub, along +The EasyBuild documentation is available at http://easybuild.readthedocs.org/. + +The EasyBuild framework source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see http://github.com/hpcugent/easybuild-framework. -The EasyBuild documentation is available on the GitHub wiki of the -easybuild meta-package, see -http://github.com/hpcugent/easybuild/wiki/Home. - -Related packages: -- easybuild-easyblocks -(http://pypi.python.org/pypi/easybuild-easyblocks): a collection of -easyblocks that implement support for building and installing (groups -of) software packages. - -- easybuild-easyconfigs -(http://pypi.python.org/pypi/easybuild-easyconfigs): a collection of -example easyconfig files that specify which software to build, and using -which build options; these easyconfigs will be well tested with the -latest compatible versions of the easybuild-framework and -easybuild-easyblocks packages. - -The code in the vsc directory originally comes from VSC-tools -(https://github.com/hpcugent/VSC-tools). +* build status - **master** branch *(Python 2.4, Python 2.6, Python 2.7)* + + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/ + +* build status - **develop** branch *(Python 2.4, Python 2.6, Python 2.7)* + + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/ + .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/badge/icon + :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/ + +Related packages: + +* `easybuild-easyblocks `_: a collection of easyblocks that implement support for building and installing (groups of) software packages. +* `easybuild-easyconfigs `_: a collection of example easyconfig files that specify which software to build, and using which build options; these easyconfigs will be well tested with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages. + +The code in the ``vsc`` directory originally comes from the *vsc-base* package +(https://github.com/hpcugent/vsc-base). From 67d4a90591d432e200281892ce5cdb4ce7e26c4c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 19:06:29 +0100 Subject: [PATCH 02/14] move build statuses to the bottom of the README --- README.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index f071e1049f..b2323a945c 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,18 @@ The EasyBuild framework source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see http://github.com/hpcugent/easybuild-framework. -* build status - **master** branch *(Python 2.4, Python 2.6, Python 2.7)* +Related packages: + +* `easybuild-easyblocks `_: a collection of easyblocks that implement support for building and installing (groups of) software packages. +* `easybuild-easyconfigs `_: a collection of example easyconfig files that specify which software to build, and using which build options; these easyconfigs will be well tested with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages. + +The code in the ``vsc`` directory originally comes from the *vsc-base* package +(https://github.com/hpcugent/vsc-base). + + +*Build status overview:* + +* **master** branch *(Python 2.4, Python 2.6, Python 2.7)* .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/badge/icon :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python24/ @@ -28,7 +39,7 @@ http://github.com/hpcugent/easybuild-framework. .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/badge/icon :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_master-python27/ -* build status - **develop** branch *(Python 2.4, Python 2.6, Python 2.7)* +* **develop** branch *(Python 2.4, Python 2.6, Python 2.7)* .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/badge/icon :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python24/ @@ -36,11 +47,3 @@ http://github.com/hpcugent/easybuild-framework. :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop/ .. image:: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/badge/icon :target: https://jenkins1.ugent.be/view/EasyBuild/job/easybuild-framework_unit-test_hpcugent_develop-python27/ - -Related packages: - -* `easybuild-easyblocks `_: a collection of easyblocks that implement support for building and installing (groups of) software packages. -* `easybuild-easyconfigs `_: a collection of example easyconfig files that specify which software to build, and using which build options; these easyconfigs will be well tested with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages. - -The code in the ``vsc`` directory originally comes from the *vsc-base* package -(https://github.com/hpcugent/vsc-base). From b7f45442301ccd54a07d6e7a5f6eb7baaf04b112 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 19:11:58 +0100 Subject: [PATCH 03/14] fix related repos links --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index b2323a945c..be8d116a59 100644 --- a/README.rst +++ b/README.rst @@ -19,10 +19,10 @@ The EasyBuild framework source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see http://github.com/hpcugent/easybuild-framework. -Related packages: +Related repositories: -* `easybuild-easyblocks `_: a collection of easyblocks that implement support for building and installing (groups of) software packages. -* `easybuild-easyconfigs `_: a collection of example easyconfig files that specify which software to build, and using which build options; these easyconfigs will be well tested with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages. +* `easybuild-easyblocks `_: a collection of easyblocks that implement support for building and installing (groups of) software packages. +* `easybuild-easyconfigs `_: a collection of example easyconfig files that specify which software to build, and using which build options; these easyconfigs will be well tested with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages. The code in the ``vsc`` directory originally comes from the *vsc-base* package (https://github.com/hpcugent/vsc-base). From 8e8942e000d57dfc83776ddf19a8a59a9d985f03 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 19:12:42 +0100 Subject: [PATCH 04/14] fix homepage URL --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index be8d116a59..418fe46b7d 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ EasyBuild: building software with ease .. image:: http://hpcugent.github.io/easybuild/images/easybuild_logo_small.png :align: center -`EasyBuild `_ is a software build +`EasyBuild `_ is a software build and installation framework that allows you to manage (scientific) software on High Performance Computing (HPC) systems in an efficient way. From a7d5b79edcb23a4e14d78ef6eee39bf486b909c3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 19:21:15 +0100 Subject: [PATCH 05/14] cosmetic changes --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 418fe46b7d..9a85c37a58 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ EasyBuild: building software with ease and installation framework that allows you to manage (scientific) software on High Performance Computing (HPC) systems in an efficient way. -The *easybuild-framework* package is the core of EasyBuild. It +The **easybuild-framework** package is the core of EasyBuild. It supports the implementation and use of so-called easyblocks which implement the software install procedure for a particular (group of) software package(s). From 054a8364daae76a043c615bbe152f00108677560 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 20:36:55 +0100 Subject: [PATCH 06/14] fix (long) description in setup.py, more cosmetics --- README.rst | 17 ++++++++++++++--- setup.py | 11 +++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 9a85c37a58..3cc0398349 100644 --- a/README.rst +++ b/README.rst @@ -19,10 +19,21 @@ The EasyBuild framework source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see http://github.com/hpcugent/easybuild-framework. -Related repositories: +Related Python packages: -* `easybuild-easyblocks `_: a collection of easyblocks that implement support for building and installing (groups of) software packages. -* `easybuild-easyconfigs `_: a collection of example easyconfig files that specify which software to build, and using which build options; these easyconfigs will be well tested with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages. +* **easybuild-easyblocks** + + * a collection of easyblocks that implement support for building and installing (groups of) software packages + * GitHub repository: http://github.com/hpcugent/easybuild-easyblocks + * package on PyPi: https://pypi.python.org/pypi/easybuild-easyblocks + +* **easybuild-easyconfigs** + + * a collection of example easyconfig files that specify which software to build, + and using which build options; these easyconfigs will be well tested + with the latest compatible versions of the easybuild-framework and easybuild-easyblocks packages + * GitHub repository: http://github.com/hpcugent/easybuild-easyconfigs + * PyPi: https://pypi.python.org/pypi/easybuild-easyconfigs The code in the ``vsc`` directory originally comes from the *vsc-base* package (https://github.com/hpcugent/vsc-base). diff --git a/setup.py b/setup.py index 5a707b3d82..119ec928e5 100644 --- a/setup.py +++ b/setup.py @@ -82,8 +82,8 @@ def find_rel_test(): version = str(VERSION), author = "EasyBuild community", author_email = "easybuild@lists.ugent.be", - description = """EasyBuild is a software installation framework in Python that allows you to \ -install software in a structured and robust way. + description = """EasyBuild is a software build and installation framework that allows you to \ +manage (scientific) software on High Performance Computing (HPC) systems in an efficient way. This package contains the EasyBuild framework, which supports the creation of custom easyblocks that \ implement support for installing particular (groups of) software packages.""", license = "GPLv2", @@ -96,12 +96,7 @@ def find_rel_test(): data_files = [ ('easybuild', ["easybuild/easybuild_config.py"]), ], - long_description = """This package contains the EasyBuild -framework, which supports the creation of custom easyblocks that -implement support for installing particular (groups of) software -packages. - -""" + read("README.rst"), + long_description = read('README.rst'), classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", From 920fffe934452352f2215220b744657c73611fde Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 20:40:01 +0100 Subject: [PATCH 07/14] fix description --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 119ec928e5..361191c59d 100644 --- a/setup.py +++ b/setup.py @@ -82,9 +82,7 @@ def find_rel_test(): version = str(VERSION), author = "EasyBuild community", author_email = "easybuild@lists.ugent.be", - description = """EasyBuild is a software build and installation framework that allows you to \ -manage (scientific) software on High Performance Computing (HPC) systems in an efficient way. -This package contains the EasyBuild framework, which supports the creation of custom easyblocks that \ + description = """The EasyBuild framework supports the creation of custom easyblocks that \ implement support for installing particular (groups of) software packages.""", license = "GPLv2", keywords = "software build building installation installing compilation HPC scientific", From a739cf1b82eb41c3a828245d86e48f6076124783 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 4 Nov 2014 20:59:47 +0100 Subject: [PATCH 08/14] drop title in README --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 3cc0398349..5b375ff615 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,3 @@ -EasyBuild: building software with ease --------------------------------------- - .. image:: http://hpcugent.github.io/easybuild/images/easybuild_logo_small.png :align: center From 7acb5749e3922cdadd1e1d53124cfe9ebdd68262 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 28 Nov 2014 18:44:41 +0100 Subject: [PATCH 09/14] support use of %(DEFAULT_ROBOT_PATHS)s template in EasyBuild configuration files --- easybuild/tools/config.py | 4 ++-- easybuild/tools/options.py | 23 +++++++++++++++-------- test/framework/config.py | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 3c4467ad3c..48ee994d62 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -387,12 +387,12 @@ def init(options, config_options_dict): def init_build_options(build_options=None, cmdline_options=None): """Initialize build options.""" - # building a dependency graph implies force, so that all dependencies are retained - # and also skips validation of easyconfigs (e.g. checking os dependencies) active_build_options = {} if cmdline_options is not None: + # building a dependency graph implies force, so that all dependencies are retained + # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if cmdline_options.dep_graph: _log.info("Enabling force to generate dependency graph.") diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 1098f59f1a..72398552a2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -75,18 +75,25 @@ class EasyBuildOptions(GeneralOption): ALLOPTSMANDATORY = False # allow more than one argument + def __init__(self, *args, **kwargs): + """Constructor.""" + + self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] + + # set up constants to seed into config files parser + go_cfg_initenv = { + 'DEFAULT': { + 'DEFAULT_ROBOT_PATHS': os.pathsep.join(self.default_robot_paths), + } + } + kwargs.setdefault('go_configfiles_initenv', {}).update(go_cfg_initenv) + super(EasyBuildOptions, self).__init__(*args, **kwargs) + def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] - easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) - if easyconfigs_pkg_paths: - default_robot_paths = easyconfigs_pkg_paths - else: - self.log.warning("basic_options: unable to determine easyconfigs pkg path for --robot-paths default") - default_robot_paths = [] - descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ @@ -100,7 +107,7 @@ def basic_options(self): 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", - 'pathlist', 'store', default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'pathlist', 'store', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), diff --git a/test/framework/config.py b/test/framework/config.py index b28ec141cf..fb4cdd854b 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -31,6 +31,7 @@ import copy import os import shutil +import sys import tempfile from test.framework.utilities import EnhancedTestCase, init_config from unittest import TestLoader @@ -42,7 +43,7 @@ from easybuild.tools.config import log_file_format, set_tmpdir, BuildOptions, ConfigurationVariables from easybuild.tools.config import get_build_log_path, DEFAULT_PATH_SUBDIRS, init_build_options, build_option from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import write_file +from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.repository.repository import init_repository @@ -443,10 +444,22 @@ def test_generaloption_config_file(self): self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + # test with config file passed via environment variable cfgtxt = '\n'.join([ '[config]', 'buildpath = %s' % testpath1, + 'robot-paths = /tmp/foo:%(DEFAULT_ROBOT_PATHS)s', ]) write_file(config_file, cfgtxt) @@ -460,6 +473,8 @@ def test_generaloption_config_file(self): self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default self.assertEqual(source_paths(), [testpath2]) # via command line self.assertEqual(build_path(), testpath1) # via config file + self.assertTrue('/tmp/foo' in options.robot_paths) + self.assertTrue(os.path.join(tmpdir, 'easybuild', 'easyconfigs') in options.robot_paths) testpath3 = os.path.join(self.tmpdir, 'testTHREE') os.environ['EASYBUILD_SOURCEPATH'] = testpath2 @@ -474,6 +489,7 @@ def test_generaloption_config_file(self): self.assertEqual(build_path(), testpath1) # via config file del os.environ['EASYBUILD_CONFIGFILES'] + sys.path[:] = orig_sys_path def test_set_tmpdir(self): """Test set_tmpdir config function.""" From 38f7895095207d16a83e0623d9c87684b551ff58 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Dec 2014 08:38:03 +0100 Subject: [PATCH 10/14] sync with vsc-base --- vsc/README.md | 2 +- vsc/utils/generaloption.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vsc/README.md b/vsc/README.md index 932369c661..b5efeb45be 100644 --- a/vsc/README.md +++ b/vsc/README.md @@ -1,3 +1,3 @@ Code from https://github.com/hpcugent/vsc-base -based on 35fee9d3130a6b52bf83993e73d187e9d46c69bc (vsc-base v1.9.9) +based on 2146be5301da34043adf4646169e5dfec88cd2f5 (vsc-base v1.9.9) diff --git a/vsc/utils/generaloption.py b/vsc/utils/generaloption.py index 390e5fa79e..e4548dc668 100644 --- a/vsc/utils/generaloption.py +++ b/vsc/utils/generaloption.py @@ -699,6 +699,7 @@ class GeneralOption(object): VERSION = None # set the version (will add --version) + DEFAULTSECT = ConfigParser.DEFAULTSECT DEFAULT_LOGLEVEL = None DEFAULT_CONFIGFILES = None DEFAULT_IGNORECONFIGFILES = None @@ -1073,7 +1074,7 @@ def configfile_parser_init(self, initenv=None): for name, section in initenv.items(): name = str(name) - if name == ConfigParser.DEFAULTSECT: + if name == self.DEFAULTSECT: # is protected/reserved (and hidden) pass elif not self.configfile_parser.has_section(name): @@ -1128,7 +1129,7 @@ def parseconfigfiles(self): self.log.debug("parseconfigfiles: following files were NOT parsed %s" % [x for x in configfiles if not x in parsed_files]) self.log.debug("parseconfigfiles: sections (w/o %s) %s" % - (ConfigParser.DEFAULTSECT, self.configfile_parser.sections())) + (self.DEFAULTSECT, self.configfile_parser.sections())) # walk through list of section names # - look for options set though config files @@ -1176,7 +1177,7 @@ def parseconfigfiles(self): actual_option = self.parser.get_option_by_long_name(opt_name) if actual_option is None: # don't fail on DEFAULT UPPERCASE options in case-sensitive mode. - in_def = self.configfile_parser.has_option(ConfigParser.DEFAULTSECT, opt) + in_def = self.configfile_parser.has_option(self.DEFAULTSECT, opt) if in_def and self.CONFIGFILE_CASESENSITIVE and opt == opt.upper(): self.log.debug(('parseconfigfiles: no option corresponding with ' 'opt %s dest %s in section %s but found all uppercase ' From 74528d96ae7d941c237a7b38905e8845734fd9d0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Dec 2014 08:38:56 +0100 Subject: [PATCH 11/14] fix remark, use DEFAULTSECT constant --- easybuild/tools/options.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 72398552a2..e21105b82d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -80,13 +80,18 @@ def __init__(self, *args, **kwargs): self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] - # set up constants to seed into config files parser - go_cfg_initenv = { - 'DEFAULT': { + # set up constants to seed into config files parser, by section + go_cfg_constants = { + GeneralOption.DEFAULTSECT: { 'DEFAULT_ROBOT_PATHS': os.pathsep.join(self.default_robot_paths), } } - kwargs.setdefault('go_configfiles_initenv', {}).update(go_cfg_initenv) + + # update or define go_configfiles_initenv in named arguments to pass to parent constructor + go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) + for section, constants in go_cfg_constants.items(): + go_cfg_initenv.setdefault(section, {}).update(constants) + super(EasyBuildOptions, self).__init__(*args, **kwargs) def basic_options(self): From 84fe650452b98d01919d7996b2ec57c78f7d2fe3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Dec 2014 10:02:36 +0100 Subject: [PATCH 12/14] add --avail-cfgfile-constants --- easybuild/tools/options.py | 34 ++++++++++++++++++++++++++++++---- test/framework/options.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e21105b82d..bd0930e6eb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -81,15 +81,17 @@ def __init__(self, *args, **kwargs): self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] # set up constants to seed into config files parser, by section - go_cfg_constants = { + self.go_cfg_constants = { GeneralOption.DEFAULTSECT: { - 'DEFAULT_ROBOT_PATHS': os.pathsep.join(self.default_robot_paths), + 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), + "List of default robot paths ('%s'-separated)" % os.pathsep), } } # update or define go_configfiles_initenv in named arguments to pass to parent constructor go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) - for section, constants in go_cfg_constants.items(): + for section, constants in self.go_cfg_constants.items(): + constants = dict([(name, value) for (name, (value, _)) in constants.items()]) go_cfg_initenv.setdefault(section, {}).update(constants) super(EasyBuildOptions, self).__init__(*args, **kwargs) @@ -268,6 +270,8 @@ def informative_options(self): descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ + 'avail-cfgfile-constants': ("Show all constants that can be used in configuration files", + None, 'store_true', False), 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", @@ -381,7 +385,7 @@ def postprocess(self): # prepare for --list/--avail if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, - self.options.list_easyblocks, self.options.list_toolchains, + self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, @@ -435,6 +439,11 @@ def _postprocess_config(self): def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' + + # dump supported configuration file constants + if self.options.avail_cfgfile_constants: + msg += self.avail_cfgfile_constants() + # dump possible easyconfig params if self.options.avail_easyconfig_params: msg += self.avail_easyconfig_params() @@ -481,6 +490,23 @@ def _postprocess_list_avail(self): print msg sys.exit(0) + def avail_cfgfile_constants(self): + """ + Return overview of constants supported in configuration files. + """ + lines = [ + "Constants available (only) in configuration files:", + "syntax: %(CONSTANT_NAME)s", + ] + for section in self.go_cfg_constants: + lines.append('') + if section != GeneralOption.DEFAULTSECT: + section_title = "only in '%s' section:" % section + lines.append(section_title) + for cst_name, (cst_value, cst_help) in sorted(self.go_cfg_constants[section].items()): + lines.append("* %s: %s [value: %s]" % (cst_name, cst_help, cst_value)) + return '\n'.join(lines) + def avail_easyconfig_params(self): """ Print the available easyconfig parameters, for the given easyblock. diff --git a/test/framework/options.py b/test/framework/options.py index 0ed83a5e74..17f3688640 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -453,6 +453,39 @@ def test_avail_lists(self): if os.path.exists(dummylogfn): os.remove(dummylogfn) + def test_avail_cfgfile_constants(self): + """Test --avail-cfgfile-constants.""" + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory + # to check whether easyconfigs install path is auto-included in robot path + tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path') + mkdir(os.path.join(tmpdir, 'easybuild'), parents=True) + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs')) + + orig_sys_path = sys.path[:] + sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs + + args = [ + '--avail-cfgfile-constants', + '--unittest-file=%s' % self.logfile, + ] + outtxt = self.eb_main(args, logfile=dummylogfn) + cfgfile_constants = { + 'DEFAULT_ROBOT_PATHS': os.path.join(tmpdir, 'easybuild', 'easyconfigs'), + } + for cst_name, cst_value in cfgfile_constants.items(): + cst_regex = re.compile("^\*\s%s:\s.*\s\[value: .*%s.*\]" % (cst_name, cst_value), re.M) + tup = (cst_regex.pattern, outtxt) + self.assertTrue(cst_regex.search(outtxt), "Pattern '%s' in --avail-cfgfile_constants output: %s" % tup) + + if os.path.exists(dummylogfn): + os.remove(dummylogfn) + sys.path[:] = orig_sys_path + def test_list_easyblocks(self): """Test listing easyblock hierarchy.""" From 9fbf6830c8cc3e752d0f41249ba4df10bb488f17 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Dec 2014 13:37:34 +0100 Subject: [PATCH 13/14] fix links to docs --- CONTRIBUTING.md | 2 +- easybuild/scripts/bootstrap_eb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10649cddab..9d866e3a8c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,7 +138,7 @@ You might also want to look into [hub](https://github.com/defunkt/hub) for more ### Review process -A member of the EasyBuild team will then review your pull request, paying attention to what you're contributing, how you implemented it and [Code style](https://github.com/hpcugent/easybuild/wiki/Code-style). +A member of the EasyBuild team will then review your pull request, paying attention to what you're contributing, how you implemented it and [code style](http://easybuild.readthedocs.org/en/latest/Code_style.html). Most likely, some remarks will be made on your pull request. Note that this is nothing personal, we're just trying to keep the EasyBuild codebase as high quality as possible. Even when an EasyBuild team member makes changes, the same public review process is followed. diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 91deb59323..3c8cc97141 100755 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -431,7 +431,7 @@ def main(): info('') info("By default, EasyBuild will install software to $HOME/.local/easybuild.") info("To install software with EasyBuild to %s, make sure $EASYBUILD_INSTALLPATH is set accordingly." % install_path) - info("See https://github.com/hpcugent/easybuild/wiki/Configuration for details on configuring EasyBuild.") + info("See http://easybuild.readthedocs.org/en/latest/Configuration.html for details on configuring EasyBuild.") # template easyconfig file for EasyBuild EB_EC_FILE = """ From 70347a2737b2960a9386cdde866637a2853770ab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Dec 2014 14:19:41 +0100 Subject: [PATCH 14/14] fix remark --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index bd0930e6eb..c2ffc8c74a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -82,7 +82,7 @@ def __init__(self, *args, **kwargs): # set up constants to seed into config files parser, by section self.go_cfg_constants = { - GeneralOption.DEFAULTSECT: { + self.DEFAULTSECT: { 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), "List of default robot paths ('%s'-separated)" % os.pathsep), } @@ -500,7 +500,7 @@ def avail_cfgfile_constants(self): ] for section in self.go_cfg_constants: lines.append('') - if section != GeneralOption.DEFAULTSECT: + if section != self.DEFAULTSECT: section_title = "only in '%s' section:" % section lines.append(section_title) for cst_name, (cst_value, cst_help) in sorted(self.go_cfg_constants[section].items()):