Merge branch 'develop' into install_extensions

easybuild/framework/easyconfig/easyconfig.py

@@ -773,6 +773,28 @@ def remove_false_versions(deps):
         # indicate that this is a parsed easyconfig
         self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
 
+    def count_files(self):
+        """
+        Determine number of files (sources + patches) required for this easyconfig.
+        """
+        cnt = len(self['sources']) + len(self['patches'])
+
+        for ext in self['exts_list']:
+            if isinstance(ext, tuple) and len(ext) >= 3:
+                ext_opts = ext[2]
+                # check for 'sources' first, since that's also considered first by EasyBlock.fetch_extension_sources
+                if 'sources' in ext_opts:
+                    cnt += len(ext_opts['sources'])
+                elif 'source_tmpl' in ext_opts:
+                    cnt += 1
+                else:
+                    # assume there's always one source file;
+                    # for extensions using PythonPackage, no 'source' or 'sources' may be specified
+                    cnt += 1
+                cnt += len(ext_opts.get('patches', []))
+
+        return cnt
+
     def local_var_naming(self, local_var_naming_check):
         """Deal with local variables that do not follow the recommended naming scheme (if any)."""
 

easybuild/main.py

@@ -68,7 +68,8 @@
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import set_up_configuration, use_color
-from easybuild.tools.output import create_progress_bar, print_checks
+from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm
+from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs
 from easybuild.tools.package.utilities import check_pkg_support
 from easybuild.tools.parallelbuild import submit_jobs
@@ -101,36 +102,30 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True, progress_bar=None):
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
     """
     Build and install software for all provided parsed easyconfig files.
 
     :param ecs: easyconfig files to install software with
     :param init_session_state: initial session state, to use in test reports
     :param exit_on_failure: whether or not to exit on installation failure
-    :param progress_bar: progress bar to use to report progress
     """
     # obtain a copy of the starting environment so each build can start afresh
     # we shouldn't use the environment from init_session_state, since relevant env vars might have been set since
     # e.g. via easyconfig.handle_allowed_system_deps
     init_env = copy.deepcopy(os.environ)
 
-    # Initialize progress bar with overall installation task
-    if progress_bar:
-        task_id = progress_bar.add_task("", total=len(ecs))
-    else:
-        task_id = None
+    start_progress_bar(STATUS_BAR, size=len(ecs))
 
     res = []
-    for ec in ecs:
+    ec_results = []
+    failed_cnt = 0
 
-        if progress_bar:
-            progress_bar.update(task_id, description=ec['short_mod_name'])
+    for ec in ecs:
 
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env, progress_bar=progress_bar,
-                                                                      task_id=task_id)
+            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
                 ec_res['err'] = EasyBuildError(err)
@@ -140,6 +135,12 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
             ec_res['err'] = err
             ec_res['traceback'] = traceback.format_exc()
 
+        if ec_res['success']:
+            ec_results.append(ec['full_mod_name'] + ' (' + colorize('OK', COLOR_GREEN) + ')')
+        else:
+            ec_results.append(ec['full_mod_name'] + ' (' + colorize('FAILED', COLOR_RED) + ')')
+            failed_cnt += 1
+
         # keep track of success/total count
         if ec_res['success']:
             test_msg = "Successfully built %s" % ec['spec']
@@ -169,6 +170,19 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True, pr
 
         res.append((ec, ec_res))
 
+        if failed_cnt:
+            # if installations failed: indicate th
+            status_label = ' (%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED)
+            failed_ecs = [x for x in ec_results[::-1] if 'FAILED' in x]
+            ok_ecs = [x for x in ec_results[::-1] if x not in failed_ecs]
+            status_label += ', '.join(failed_ecs + ok_ecs)
+        else:
+            status_label = ': ' + ', '.join(ec_results[::-1])
+
+        update_progress_bar(STATUS_BAR, label=status_label)
+
+    stop_progress_bar(STATUS_BAR)
+
     return res
 
 
@@ -540,11 +554,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
 
-        progress_bar = create_progress_bar()
-        with progress_bar:
+        with rich_live_cm():
             ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
-                                                      exit_on_failure=exit_on_failure,
-                                                      progress_bar=progress_bar)
+                                                      exit_on_failure=exit_on_failure)
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 

easybuild/tools/filetools.py

@@ -54,13 +54,15 @@
 import tempfile
 import time
 import zlib
+from functools import partial
 
 from easybuild.base import fancylogger
 from easybuild.tools import run
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
 from easybuild.tools.config import build_option, install_path
+from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
 from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars
 
@@ -215,7 +217,8 @@ def read_file(path, log_error=True, mode='r'):
     return txt
 
 
-def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False):
+def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False,
+               show_progress=False, size=None):
     """
     Write given contents to file at given path;
     overwrites current file contents without backup by default!
@@ -227,6 +230,8 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     :param backup: back up existing file before overwriting or modifying it
     :param always_overwrite: don't require --force to overwrite an existing file
     :param verbose: be verbose, i.e. inform where backup file was created
+    :param show_progress: show progress bar while writing file
+    :param size: size (in bytes) of data to write (used for progress bar)
     """
     # early exit in 'dry run' mode
     if not forced and build_option('extended_dry_run'):
@@ -256,15 +261,30 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over
     if sys.version_info[0] >= 3 and (isinstance(data, bytes) or data_is_file_obj):
         mode += 'b'
 
+    # don't bother showing a progress bar for small files (< 10MB)
+    if size and size < 10 * (1024 ** 2):
+        _log.info("Not showing progress bar for downloading small file (size %s)", size)
+        show_progress = False
+
+    if show_progress:
+        start_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, size, label=os.path.basename(path))
+
     # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
     try:
         mkdir(os.path.dirname(path), parents=True)
         with open_file(path, mode) as fh:
             if data_is_file_obj:
-                # if a file-like object was provided, use copyfileobj (which reads the file in chunks)
-                shutil.copyfileobj(data, fh)
+                # if a file-like object was provided, read file in 1MB chunks
+                for chunk in iter(partial(data.read, 1024 ** 2), b''):
+                    fh.write(chunk)
+                    if show_progress:
+                        update_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE, progress_size=len(chunk))
             else:
                 fh.write(data)
+
+        if show_progress:
+            stop_progress_bar(PROGRESS_BAR_DOWNLOAD_ONE)
+
     except IOError as err:
         raise EasyBuildError("Failed to write to %s: %s", path, err)
 
@@ -701,6 +721,22 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
     return urlpat_headers
 
 
+def det_file_size(http_header):
+    """
+    Determine size of file from provided HTTP header info (without downloading it).
+    """
+    res = None
+    len_key = 'Content-Length'
+    if len_key in http_header:
+        size = http_header[len_key]
+        try:
+            res = int(size)
+        except (ValueError, TypeError) as err:
+            _log.warning("Failed to interpret size '%s' as integer value: %s", size, err)
+
+    return res
+
+
 def download_file(filename, url, path, forced=False):
     """Download a file from the given URL, to the specified path."""
 
@@ -757,19 +793,24 @@ def download_file(filename, url, path, forced=False):
                 # urllib does not!
                 url_fd = std_urllib.urlopen(url_req, timeout=timeout)
                 status_code = url_fd.getcode()
+                size = det_file_size(url_fd.info())
             else:
                 response = requests.get(url, headers=headers, stream=True, timeout=timeout)
                 status_code = response.status_code
                 response.raise_for_status()
+                size = det_file_size(response.headers)
                 url_fd = response.raw
                 url_fd.decode_content = True
-            _log.debug('response code for given url %s: %s' % (url, status_code))
+
+            _log.debug("HTTP response code for given url %s: %s", url, status_code)
+            _log.info("File size for %s: %s", url, size)
+
             # note: we pass the file object to write_file rather than reading the file first,
             # to ensure the data is read in chunks (which prevents problems in Python 3.9+);
             # cfr. https://github.com/easybuilders/easybuild-framework/issues/3455
             # and https://bugs.python.org/issue42853
-            write_file(path, url_fd, forced=forced, backup=True)
-            _log.info("Downloaded file %s from url %s to %s" % (filename, url, path))
+            write_file(path, url_fd, forced=forced, backup=True, show_progress=True, size=size)
+            _log.info("Downloaded file %s from url %s to %s", filename, url, path)
             downloaded = True
             url_fd.close()
         except used_urllib.HTTPError as err:
@@ -1409,6 +1450,44 @@ def guess_patch_level(patched_files, parent_dir):
     return patch_level
 
 
+def create_patch_info(patch_spec):
+    """
+    Create info dictionary from specified patch spec.
+    """
+    if isinstance(patch_spec, (list, tuple)):
+        if not len(patch_spec) == 2:
+            error_msg = "Unknown patch specification '%s', only 2-element lists/tuples are supported!"
+            raise EasyBuildError(error_msg, str(patch_spec))
+
+        patch_info = {'name': patch_spec[0]}
+
+        patch_arg = patch_spec[1]
+        # patch level *must* be of type int, nothing else (not True/False!)
+        # note that 'isinstance(..., int)' returns True for True/False values...
+        if isinstance(patch_arg, int) and not isinstance(patch_arg, bool):
+            patch_info['level'] = patch_arg
+
+        # string value as patch argument can be either path where patch should be applied,
+        # or path to where a non-patch file should be copied
+        elif isinstance(patch_arg, string_type):
+            if patch_spec[0].endswith('.patch'):
+                patch_info['sourcepath'] = patch_arg
+            # non-patch files are assumed to be files to copy
+            else:
+                patch_info['copy'] = patch_arg
+        else:
+            raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element",
+                                 str(patch_spec))
+
+    elif isinstance(patch_spec, string_type):
+        patch_info = {'name': patch_spec}
+    else:
+        error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s"
+        raise EasyBuildError(error_msg, patch_spec)
+
+    return patch_info
+
+
 def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False):
     """
     Apply a patch to source code in directory dest
@@ -2290,10 +2369,13 @@ def copy_file(path, target_path, force_in_dry_run=False):
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
                 if os.path.islink(path):
+                    if os.path.isdir(target_path):
+                        target_path = os.path.join(target_path, os.path.basename(path))
+                        _log.info("target_path changed to %s", target_path)
                     # special care for copying broken symlinks
                     link_target = os.readlink(path)
-                    symlink(link_target, target_path)
-                    _log.info("created symlink from %s to %s", path, target_path)
+                    symlink(link_target, target_path, use_abspath_source=False)
+                    _log.info("created symlink %s to %s", link_target, target_path)
                 else:
                     shutil.copy2(path, target_path)
                     _log.info("%s copied to %s", path, target_path)