Merge branch 'develop' into lock_cleanup

easybuild/framework/easyblock.py

@@ -2409,37 +2409,71 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
             SANITY_CHECK_PATHS_DIRS: ("(non-empty) directory", lambda dp: os.path.isdir(dp) and os.listdir(dp)),
         }
 
-        # prepare sanity check paths
-        paths = self.cfg['sanity_check_paths']
-        if not paths:
+        enhance_sanity_check = self.cfg['enhance_sanity_check']
+        ec_commands = self.cfg['sanity_check_commands']
+        ec_paths = self.cfg['sanity_check_paths']
+
+        # if enhance_sanity_check is not enabled, only sanity_check_paths specified in the easyconfig file are used,
+        # the ones provided by the easyblock (via custom_paths) are ignored
+        if ec_paths and not enhance_sanity_check:
+            paths = ec_paths
+            self.log.info("Using (only) sanity check paths specified by easyconfig file: %s", paths)
+        else:
+            # if no sanity_check_paths are specified in easyconfig,
+            # we fall back to the ones provided by the easyblock via custom_paths
             if custom_paths:
                 paths = custom_paths
-                self.log.info("Using customized sanity check paths: %s" % paths)
+                self.log.info("Using customized sanity check paths: %s", paths)
+            # if custom_paths is empty, we fall back to a generic set of paths:
+            # non-empty bin/ + /lib or /lib64 directories
             else:
                 paths = {}
                 for key in path_keys_and_check:
                     paths.setdefault(key, [])
                 paths.update({SANITY_CHECK_PATHS_DIRS: ['bin', ('lib', 'lib64')]})
-                self.log.info("Using default sanity check paths: %s" % paths)
+                self.log.info("Using default sanity check paths: %s", paths)
+
+            # if enhance_sanity_check is enabled *and* sanity_check_paths are specified in the easyconfig,
+            # those paths are used to enhance the paths provided by the easyblock
+            if enhance_sanity_check and ec_paths:
+                for key in ec_paths:
+                    val = ec_paths[key]
+                    if isinstance(val, list):
+                        paths[key] = paths.get(key, []) + val
+                    else:
+                        error_pattern = "Incorrect value type in sanity_check_paths, should be a list: "
+                        error_pattern += "%s (type: %s)" % (val, type(val))
+                        raise EasyBuildError(error_pattern)
+                self.log.info("Enhanced sanity check paths after taking into account easyconfig file: %s", paths)
+
+        sorted_keys = sorted(paths.keys())
+        known_keys = sorted(path_keys_and_check.keys())
+
+        # verify sanity_check_paths value: only known keys, correct value types, at least one non-empty value
+        only_list_values = all(isinstance(x, list) for x in paths.values())
+        only_empty_lists = all(not x for x in paths.values())
+        if sorted_keys != known_keys or not only_list_values or only_empty_lists:
+            error_msg = "Incorrect format for sanity_check_paths: should (only) have %s keys, "
+            error_msg += "values should be lists (at least one non-empty)."
+            raise EasyBuildError(error_msg % ', '.join("'%s'" % k for k in known_keys))
+
+        # if enhance_sanity_check is not enabled, only sanity_check_commands specified in the easyconfig file are used,
+        # the ones provided by the easyblock (via custom_commands) are ignored
+        if ec_commands and not enhance_sanity_check:
+            commands = ec_commands
+            self.log.info("Using (only) sanity check commands specified by easyconfig file: %s", commands)
         else:
-            self.log.info("Using specified sanity check paths: %s" % paths)
-
-        ks = sorted(paths.keys())
-        valnottypes = [not isinstance(x, list) for x in paths.values()]
-        lenvals = [len(x) for x in paths.values()]
-        req_keys = sorted(path_keys_and_check.keys())
-        if not ks == req_keys or sum(valnottypes) > 0 or sum(lenvals) == 0:
-            raise EasyBuildError("Incorrect format for sanity_check_paths (should (only) have %s keys, "
-                                 "values should be lists (at least one non-empty)).", ','.join(req_keys))
-
-        commands = self.cfg['sanity_check_commands']
-        if not commands:
             if custom_commands:
                 commands = custom_commands
-                self.log.info("Using customised sanity check commands: %s" % commands)
+                self.log.info("Using customised sanity check commands: %s", commands)
             else:
                 commands = []
-                self.log.info("Using specified sanity check commands: %s" % commands)
+
+            # if enhance_sanity_check is enabled, the sanity_check_commands specified in the easyconfig file
+            # are combined with those provided by the easyblock via custom_commands
+            if enhance_sanity_check and ec_commands:
+                commands = commands + ec_commands
+                self.log.info("Enhanced sanity check commands after taking into account easyconfig file: %s", commands)
 
         for i, command in enumerate(commands):
             # set command to default. This allows for config files with
@@ -2475,9 +2509,17 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, **
         """
         paths, path_keys_and_check, commands = self._sanity_check_step_common(custom_paths, custom_commands)
 
-        for key, (typ, _) in path_keys_and_check.items():
+        for key in [SANITY_CHECK_PATHS_FILES, SANITY_CHECK_PATHS_DIRS]:
+            (typ, _) = path_keys_and_check[key]
             self.dry_run_msg("Sanity check paths - %s ['%s']", typ, key)
-            if paths[key]:
+            entries = paths[key]
+            if entries:
+                # some entries may be tuple values,
+                # we need to convert them to strings first so we can print them sorted
+                for idx, entry in enumerate(entries):
+                    if isinstance(entry, tuple):
+                        entries[idx] = ' or '.join(entry)
+
                 for path in sorted(paths[key]):
                     self.dry_run_msg("  * %s", str(path))
             else:
@@ -2608,6 +2650,9 @@ def xs2str(xs):
 
         # run sanity check commands
         for command in commands:
+
+            trace_msg("running command '%s' ..." % command)
+
             out, ec = run_cmd(command, simple=False, log_ok=False, log_all=False, trace=False)
             if ec != 0:
                 fail_msg = "sanity check command %s exited with code %s (output: %s)" % (command, ec, out)
@@ -2616,7 +2661,7 @@ def xs2str(xs):
             else:
                 self.log.info("sanity check command %s ran successfully! (output: %s)" % (command, out))
 
-            trace_msg("running command '%s': %s" % (command, ('FAILED', 'OK')[ec == 0]))
+            trace_msg("result for command '%s': %s" % (command, ('FAILED', 'OK')[ec == 0]))
 
         # also run sanity check for extensions (unless we are an extension ourselves)
         if not extension:

easybuild/framework/easyconfig/default.py

@@ -90,6 +90,8 @@
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],
+    'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance "
+                             "the existin sanity check, not replace it", BUILD],
     'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed "
                                    "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD],
     'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed "

easybuild/framework/easyconfig/format/one.py

@@ -97,6 +97,13 @@ class FormatOneZero(EasyConfigFormatConfigObj):
     PYHEADER_MANDATORY = ['version', 'name', 'toolchain', 'homepage', 'description']
     PYHEADER_BLACKLIST = []
 
+    def __init__(self, *args, **kwargs):
+        """FormatOneZero constructor."""
+        super(FormatOneZero, self).__init__(*args, **kwargs)
+
+        self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
+        self.strict_sanity_check_paths_keys = True
+
     def validate(self):
         """Format validation"""
         # minimal checks
@@ -168,11 +175,14 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
                     for item_key in ordered_item_keys:
                         if item_key in param_val:
                             item_val = param_val[item_key]
+                            item_comments = self._get_item_comments(param_name, item_val)
+                        elif param_name == 'sanity_check_paths' and not self.strict_sanity_check_paths_keys:
+                            item_val = []
+                            item_comments = {}
+                            self.log.info("Using default value for '%s' in sanity_check_paths: %s", item_key, item_val)
                         else:
                             raise EasyBuildError("Missing mandatory key '%s' in %s.", item_key, param_name)
 
-                        item_comments = self._get_item_comments(param_name, item_val)
-
                         inline_comment = item_comments.get('inline', '')
                         item_tmpl_dict = {'inline_comment': inline_comment}
 
@@ -317,6 +327,10 @@ def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy
         :param templ_val: known template values
         :param toolchain_hierarchy: hierarchy of toolchains for easyconfig
         """
+        # figoure out whether we should be strict about the format of sanity_check_paths;
+        # if enhance_sanity_check is set, then both files/dirs keys are not strictly required...
+        self.strict_sanity_check_paths_keys = not ecfg['enhance_sanity_check']
+
         # include header comments first
         dump = self.comments['header'][:]
 

easybuild/tools/filetools.py

@@ -40,7 +40,6 @@
 """
 import datetime
 import difflib
-import distutils.dir_util
 import fileinput
 import glob
 import hashlib
@@ -499,7 +498,13 @@ def pypi_source_urls(pkg_name):
         _log.debug("Failed to download %s to determine available PyPI URLs for %s", simple_url, pkg_name)
         res = []
     else:
-        parsed_html = ElementTree.parse(urls_html)
+        urls_txt = read_file(urls_html)
+
+        # ignore yanked releases (see https://pypi.org/help/#yanked)
+        # see https://github.com/easybuilders/easybuild-framework/issues/3301
+        urls_txt = re.sub(r'<a.*?data-yanked.*?</a>', '', urls_txt)
+
+        parsed_html = ElementTree.ElementTree(ElementTree.fromstring(urls_txt))
         if hasattr(parsed_html, 'iter'):
             res = [a.attrib['href'] for a in parsed_html.iter('a')]
         else:
@@ -762,6 +767,18 @@ def find_easyconfigs(path, ignore_dirs=None):
     return files
 
 
+def find_glob_pattern(glob_pattern, fail_on_no_match=True):
+    """Find unique file/dir matching glob_pattern (raises error if more than one match is found)"""
+    if build_option('extended_dry_run'):
+        return glob_pattern
+    res = glob.glob(glob_pattern)
+    if len(res) == 0 and not fail_on_no_match:
+        return None
+    if len(res) != 1:
+        raise EasyBuildError("Was expecting exactly one match for '%s', found %d: %s", glob_pattern, len(res), res)
+    return res[0]
+
+
 def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False,
                 case_sensitive=False):
     """
@@ -2008,7 +2025,12 @@ def copy_file(path, target_path, force_in_dry_run=False):
                 _log.info("Copied contents of file %s to %s", path, target_path)
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
-                shutil.copy2(path, target_path)
+                if os.path.exists(path):
+                    shutil.copy2(path, target_path)
+                elif os.path.islink(path):
+                    # special care for copying broken symlinks
+                    link_target = os.readlink(path)
+                    symlink(link_target, target_path)
                 _log.info("%s copied to %s", path, target_path)
         except (IOError, OSError, shutil.Error) as err:
             raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
@@ -2043,16 +2065,13 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k
     :param path: the original directory path
     :param target_path: path to copy the directory to
     :param force_in_dry_run: force running the command during dry run
-    :param dirs_exist_ok: wrapper around shutil.copytree option, which was added in Python 3.8
+    :param dirs_exist_ok: boolean indicating whether it's OK if the target directory already exists
 
-    On Python >= 3.8 shutil.copytree is always used
-    On Python < 3.8 if 'dirs_exist_ok' is False - shutil.copytree is used
-    On Python < 3.8 if 'dirs_exist_ok' is True - distutils.dir_util.copy_tree is used
+    shutil.copytree is used if the target path does not exist yet;
+    if the target path already exists, the 'copy' function will be used to copy the contents of
+    the source path to the target path
 
-    Additional specified named arguments are passed down to shutil.copytree if used.
-
-    Because distutils.dir_util.copy_tree supports only 'symlinks' named argument,
-    using any other will raise EasyBuildError.
+    Additional specified named arguments are passed down to shutil.copytree/copy if used.
     """
     if not force_in_dry_run and build_option('extended_dry_run'):
         dry_run_msg("copied directory %s to %s" % (path, target_path))
@@ -2061,38 +2080,49 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k
             if not dirs_exist_ok and os.path.exists(target_path):
                 raise EasyBuildError("Target location %s to copy %s to already exists", target_path, path)
 
-            if sys.version_info >= (3, 8):
-                # on Python >= 3.8, shutil.copytree works fine, thanks to availability of dirs_exist_ok named argument
-                shutil.copytree(path, target_path, dirs_exist_ok=dirs_exist_ok, **kwargs)
+            # note: in Python >= 3.8 shutil.copytree works just fine thanks to the 'dirs_exist_ok' argument,
+            # but since we need to be more careful in earlier Python versions we use our own implementation
+            # in case the target directory exists and 'dirs_exist_ok' is enabled
+            if dirs_exist_ok and os.path.exists(target_path):
+                # if target directory already exists (and that's allowed via dirs_exist_ok),
+                # we need to be more careful, since shutil.copytree will fail (in Python < 3.8)
+                # if target directory already exists;
+                # so, recurse via 'copy' function to copy files/dirs in source path to target path
+                # (NOTE: don't use distutils.dir_util.copy_tree here, see
+                # https://github.com/easybuilders/easybuild-framework/issues/3306)
+
+                entries = os.listdir(path)
 
-            elif dirs_exist_ok:
-                # use distutils.dir_util.copy_tree with Python < 3.8 if dirs_exist_ok is enabled
+                # take into account 'ignore' function that is supported by shutil.copytree
+                # (but not by 'copy_file' function used by 'copy')
+                ignore = kwargs.get('ignore')
+                if ignore:
+                    ignored_entries = ignore(path, entries)
+                    entries = [x for x in entries if x not in ignored_entries]
 
-                # first get value for symlinks named argument (if any)
-                preserve_symlinks = kwargs.pop('symlinks', False)
+                # determine list of paths to copy
+                paths_to_copy = [os.path.join(path, x) for x in entries]
 
-                # check if there are other named arguments (there shouldn't be, only 'symlinks' is supported)
-                if kwargs:
-                    raise EasyBuildError("Unknown named arguments passed to copy_dir with dirs_exist_ok=True: %s",
-                                         ', '.join(sorted(kwargs.keys())))
-                distutils.dir_util.copy_tree(path, target_path, preserve_symlinks=preserve_symlinks)
+                copy(paths_to_copy, target_path,
+                     force_in_dry_run=force_in_dry_run, dirs_exist_ok=dirs_exist_ok, **kwargs)
 
             else:
-                # if dirs_exist_ok is not enabled, just use shutil.copytree
+                # if dirs_exist_ok is not enabled or target directory doesn't exist, just use shutil.copytree
                 shutil.copytree(path, target_path, **kwargs)
 
             _log.info("%s copied to %s", path, target_path)
-        except (IOError, OSError) as err:
+        except (IOError, OSError, shutil.Error) as err:
             raise EasyBuildError("Failed to copy directory %s to %s: %s", path, target_path, err)
 
 
-def copy(paths, target_path, force_in_dry_run=False):
+def copy(paths, target_path, force_in_dry_run=False, **kwargs):
     """
     Copy single file/directory or list of files and directories to specified location
 
     :param paths: path(s) to copy
     :param target_path: target location
     :param force_in_dry_run: force running the command during dry run
+    :param kwargs: additional named arguments to pass down to copy_dir
     """
     if isinstance(paths, string_type):
         paths = [paths]
@@ -2103,10 +2133,11 @@ def copy(paths, target_path, force_in_dry_run=False):
         full_target_path = os.path.join(target_path, os.path.basename(path))
         mkdir(os.path.dirname(full_target_path), parents=True)
 
-        if os.path.isfile(path):
+        # copy broken symlinks only if 'symlinks=True' is used
+        if os.path.isfile(path) or (os.path.islink(path) and kwargs.get('symlinks')):
             copy_file(path, full_target_path, force_in_dry_run=force_in_dry_run)
         elif os.path.isdir(path):
-            copy_dir(path, full_target_path, force_in_dry_run=force_in_dry_run)
+            copy_dir(path, full_target_path, force_in_dry_run=force_in_dry_run, **kwargs)
         else:
             raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path)