add support for --extended-dry-run/-x (REVIEW)

easybuild/framework/easyconfig/easyconfig.py

@@ -484,7 +484,21 @@ def toolchain(self):
         returns the Toolchain used
         """
         if self._toolchain is None:
-            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS())
+            # provide list of (direct) toolchain dependencies (name & version), if easyconfig can be found for toolchain
+            tcdeps = None
+            tcname, tcversion = self['toolchain']['name'], self['toolchain']['version']
+            if tcname != DUMMY_TOOLCHAIN_NAME:
+                tc_ecfile = robot_find_easyconfig(tcname, tcversion)
+                if tc_ecfile is None:
+                    self.log.debug("No easyconfig found for toolchain %s version %s, can't determine dependencies",
+                                   tcname, tcversion)
+                else:
+                    self.log.debug("Found easyconfig for toolchain %s version %s: %s", tcname, tcversion, tc_ecfile)
+                    tc_ec = process_easyconfig(tc_ecfile)[0]
+                    tcdeps = tc_ec['dependencies']
+                    self.log.debug("Toolchain dependencies based on easyconfig: %s", tcdeps)
+
+            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS(), tcdeps=tcdeps)
             tc_dict = self._toolchain.as_dict()
             self.log.debug("Initialized toolchain: %s (opts: %s)" % (tc_dict, self['toolchainopts']))
         return self._toolchain
@@ -1014,28 +1028,33 @@ def create_paths(path, name, version):
 
 def robot_find_easyconfig(name, version):
     """
-    Find an easyconfig for module in path
+    Find an easyconfig for module in path, returns (absolute) path to easyconfig file (or None, if none is found).
     """
     key = (name, version)
     if key in _easyconfig_files_cache:
         _log.debug("Obtained easyconfig path from cache for %s: %s" % (key, _easyconfig_files_cache[key]))
         return _easyconfig_files_cache[key]
+
     paths = build_option('robot_path')
-    if not paths:
-        raise EasyBuildError("No robot path specified, which is required when looking for easyconfigs (use --robot)")
-    if not isinstance(paths, (list, tuple)):
+    if paths is None:
+        paths = []
+    elif not isinstance(paths, (list, tuple)):
         paths = [paths]
-    # candidate easyconfig paths
+
+    res = None
     for path in paths:
         easyconfigs_paths = create_paths(path, name, version)
         for easyconfig_path in easyconfigs_paths:
             _log.debug("Checking easyconfig path %s" % easyconfig_path)
             if os.path.isfile(easyconfig_path):
                 _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path))
                 _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path)
-                return _easyconfig_files_cache[key]
+                res = _easyconfig_files_cache[key]
+                break
+        if res:
+            break
 
-    return None
+    return res
 
 
 class ActiveMNS(object):

easybuild/framework/extension.py

@@ -37,7 +37,7 @@
 import os
 
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import build_path
+from easybuild.tools.config import build_option, build_path
 from easybuild.tools.run import run_cmd
 
 
@@ -62,7 +62,9 @@ def __init__(self, mself, ext):
         self.patches = self.ext.get('patches', [])
         self.options = copy.deepcopy(self.ext.get('options', {}))
 
-        self.toolchain.prepare(self.cfg['onlytcmod'])
+        # don't re-prepare the build environment when doing a dry run, since it'll be the same as for the parent
+        if not build_option('extended_dry_run'):
+            self.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True)
 
         self.sanity_check_fail_msgs = []
 

easybuild/main.py

@@ -129,7 +129,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         # dump test report next to log file
         test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state)
-        if 'log_file' in ec_res:
+        if 'log_file' in ec_res and ec_res['log_file']:
             test_report_fp = "%s_test_report.md" % '.'.join(ec_res['log_file'].split('.')[:-1])
             parent_dir = os.path.dirname(test_report_fp)
             # parent dir for test report may not be writable at this time, e.g. when --read-only-installdir is used
@@ -292,7 +292,7 @@ def main(args=None, logfile=None, do_build=None, testing=False):
         sys.exit(0)
 
     # skip modules that are already installed unless forced
-    if not options.force:
+    if not (options.force or options.extended_dry_run):
         retained_ecs = skip_available(easyconfigs)
         if not testing:
             for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:

easybuild/toolchains/fft/intelfftw.py

@@ -31,7 +31,8 @@
 import os
 from distutils.version import LooseVersion
 
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, dry_run_warning
+from easybuild.tools.config import build_option
 from easybuild.toolchains.fft.fftw import Fftw
 from easybuild.tools.modules import get_software_root, get_software_version
 
@@ -90,5 +91,9 @@ def _set_fftw_variables(self):
         if all([fftw_lib_exists(lib) for lib in check_fftw_libs]):
             self.FFT_LIB = fftw_libs
         else:
-            raise EasyBuildError("Not all FFTW interface libraries %s are found in %s, can't set FFT_LIB.",
-                                 check_fftw_libs, fft_lib_dirs)
+            msg = "Not all FFTW interface libraries %s are found in %s" % (check_fftw_libs, fft_lib_dirs)
+            msg += ", can't set $FFT_LIB."
+            if build_option('extended_dry_run'):
+                dry_run_warning(msg, silent=build_option('silent'))
+            else:
+                raise EasyBuildError(msg)

easybuild/tools/build_log.py

@@ -32,6 +32,7 @@
 @author: Jens Timmerman (Ghent University)
 """
 import os
+import re
 import sys
 import tempfile
 from copy import copy
@@ -52,6 +53,10 @@
 
 DEPRECATED_DOC_URL = 'http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html'
 
+DRY_RUN_BUILD_DIR = None
+DRY_RUN_SOFTWARE_INSTALL_DIR = None
+DRY_RUN_MODULES_INSTALL_DIR = None
+
 
 class EasyBuildError(LoggedException):
     """
@@ -225,6 +230,42 @@ def print_msg(msg, log=None, silent=False, prefix=True):
             print msg
 
 
+def dry_run_set_dirs(prefix, builddir, software_installdir, module_installdir):
+    """
+    Initialize for printing dry run messages.
+
+    Define DRY_RUN_*DIR constants, so they can be used in dry_run_msg to replace fake build/install dirs.
+
+    @param prefix: prefix of fake build/install dirs, that can be stripped off when printing
+    @param builddir: fake build dir
+    @param software_installdir: fake software install directory
+    @param module_installdir: fake module install directory
+    """
+    global DRY_RUN_BUILD_DIR
+    DRY_RUN_BUILD_DIR = (re.compile(builddir), builddir[len(prefix):])
+
+    global DRY_RUN_MODULES_INSTALL_DIR
+    DRY_RUN_MODULES_INSTALL_DIR = (re.compile(module_installdir), module_installdir[len(prefix):])
+
+    global DRY_RUN_SOFTWARE_INSTALL_DIR
+    DRY_RUN_SOFTWARE_INSTALL_DIR = (re.compile(software_installdir), software_installdir[len(prefix):])
+
+
+def dry_run_msg(msg, silent=False):
+    """Print dry run message."""
+    # replace fake build/install dir in dry run message with original value
+    for dry_run_var in [DRY_RUN_BUILD_DIR, DRY_RUN_MODULES_INSTALL_DIR, DRY_RUN_SOFTWARE_INSTALL_DIR]:
+        if dry_run_var is not None:
+            msg = dry_run_var[0].sub(dry_run_var[1], msg)
+
+    print_msg(msg, silent=silent, prefix=False)
+
+
+def dry_run_warning(msg, silent=False):
+    """Print dry run message."""
+    dry_run_msg("\n!!!\n!!! WARNING: %s\n!!!\n" % msg, silent=silent)
+
+
 def print_error(message, log=None, exitCode=1, opt_parser=None, exit_on_error=True, silent=False):
     """
     Print error message and exit EasyBuild

easybuild/tools/config.py

@@ -43,11 +43,8 @@
 from vsc.utils.missing import FrozenDictKnownKeys
 from vsc.utils.patterns import Singleton
 
-import easybuild.tools.environment as env
-from easybuild.tools import run
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.module_naming_scheme import GENERAL_CLASS
-from easybuild.tools.run import run_cmd
 
 
 _log = fancylogger.getLogger('config', fname=False)
@@ -77,7 +74,6 @@
 DEFAULT_PNS = 'EasyBuildPNS'
 DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild")
 DEFAULT_REPOSITORY = 'FileRepository'
-DEFAULT_STRICT = run.WARN
 
 
 # utility function for obtaining default paths
@@ -124,6 +120,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'allow_modules_tool_mismatch',
         'debug',
         'dump_autopep8',
+        'extended_dry_run',
         'experimental',
         'force',
         'group_writable_installdir',
@@ -142,8 +139,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     True: [
         'cleanup_builddir',
         'cleanup_tmpdir',
+        'extended_dry_run_ignore_errors',
     ],
-    DEFAULT_STRICT: [
+    'warn': [
         'strict',
     ],
     DEFAULT_PKG_RELEASE: [
@@ -303,7 +301,9 @@ def init_build_options(build_options=None, cmdline_options=None):
             cmdline_options.force = True
             retain_all_deps = True
 
-        if cmdline_options.dep_graph or cmdline_options.dry_run or cmdline_options.dry_run_short:
+        auto_ignore_osdeps_options = [cmdline_options.dep_graph, cmdline_options.dry_run, cmdline_options.dry_run_short,
+                                      cmdline_options.extended_dry_run]
+        if any(auto_ignore_osdeps_options):
             _log.info("Ignoring OS dependencies for --dep-graph/--dry-run")
             cmdline_options.ignore_osdeps = True
 
@@ -333,9 +333,15 @@ def init_build_options(build_options=None, cmdline_options=None):
     return BuildOptions(bo)
 
 
-def build_option(key):
+def build_option(key, **kwargs):
     """Obtain value specified build option."""
-    return BuildOptions()[key]
+    build_options = BuildOptions()
+    if key in build_options:
+        return build_options[key]
+    elif 'default' in kwargs:
+        return kwargs['default']
+    else:
+        raise EasyBuildError("Undefined build option: %s", key)
 
 
 def build_path():
@@ -517,46 +523,3 @@ def module_classes():
 def read_environment(env_vars, strict=False):
     """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead"""
     _log.nosupport("read_environment has moved to easybuild.tools.environment", '2.0')
-
-
-def set_tmpdir(tmpdir=None, raise_error=False):
-    """Set temporary directory to be used by tempfile and others."""
-    try:
-        if tmpdir is not None:
-            if not os.path.exists(tmpdir):
-                os.makedirs(tmpdir)
-            current_tmpdir = tempfile.mkdtemp(prefix='eb-', dir=tmpdir)
-        else:
-            # use tempfile default parent dir
-            current_tmpdir = tempfile.mkdtemp(prefix='eb-')
-    except OSError, err:
-        raise EasyBuildError("Failed to create temporary directory (tmpdir: %s): %s", tmpdir, err)
-
-    _log.info("Temporary directory used in this EasyBuild run: %s" % current_tmpdir)
-
-    for var in ['TMPDIR', 'TEMP', 'TMP']:
-        env.setvar(var, current_tmpdir)
-
-    # reset to make sure tempfile picks up new temporary directory to use
-    tempfile.tempdir = None
-
-    # test if temporary directory allows to execute files, warn if it doesn't
-    try:
-        fd, tmptest_file = tempfile.mkstemp()
-        os.close(fd)
-        os.chmod(tmptest_file, 0700)
-        if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False):
-            msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir()
-            msg += "This can cause problems in the build process, consider using --tmpdir."
-            if raise_error:
-                raise EasyBuildError(msg)
-            else:
-                _log.warning(msg)
-        else:
-            _log.debug("Temporary directory %s allows to execute files, good!" % tempfile.gettempdir())
-        os.remove(tmptest_file)
-
-    except OSError, err:
-        raise EasyBuildError("Failed to test whether temporary directory allows to execute files: %s", err)
-
-    return current_tmpdir

easybuild/tools/environment.py

@@ -33,7 +33,8 @@
 from vsc.utils import fancylogger
 from vsc.utils.missing import shell_quote
 
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, dry_run_msg
+from easybuild.tools.config import build_option
 
 
 # take copy of original environemt, so we can restore (parts of) it later
@@ -79,10 +80,12 @@ def get_changes():
     return _changes
 
 
-def setvar(key, value):
+def setvar(key, value, verbose=True):
     """
     put key in the environment with value
     tracks added keys until write_changes has been called
+
+    @param verbose: include message in dry run output for defining this environment variable
     """
     if key in os.environ:
         oldval_info = "previous value: '%s'" % os.environ[key]
@@ -93,6 +96,12 @@ def setvar(key, value):
     _changes[key] = value
     _log.info("Environment variable %s set to %s (%s)", key, value, oldval_info)
 
+    if verbose and build_option('extended_dry_run'):
+        quoted_value = shell_quote(value)
+        if quoted_value[0] not in ['"', "'"]:
+            quoted_value = '"%s"' % quoted_value
+        dry_run_msg("  export %s=%s" % (key, quoted_value), silent=build_option('silent'))
+
 
 def unset_env_vars(keys):
     """
@@ -139,7 +148,7 @@ def read_environment(env_vars, strict=False):
     return result
 
 
-def modify_env(old, new):
+def modify_env(old, new, verbose=True):
     """
     Compares 2 os.environ dumps. Adapts final environment.
     """
@@ -151,10 +160,10 @@ def modify_env(old, new):
             ## hmm, smart checking with debug logging
             if not new[key] == old[key]:
                 _log.debug("Key in new environment found that is different from old one: %s (%s)" % (key, new[key]))
-                setvar(key, new[key])
+                setvar(key, new[key], verbose=verbose)
         else:
             _log.debug("Key in new environment found that is not in old one: %s (%s)" % (key, new[key]))
-            setvar(key, new[key])
+            setvar(key, new[key], verbose=verbose)
 
     for key in oldKeys:
         if not key in newKeys:
@@ -167,4 +176,4 @@ def restore_env(env):
     """
     Restore active environment based on specified dictionary.
     """
-    modify_env(os.environ, env)
+    modify_env(os.environ, env, verbose=False)