Merge pull request #570 from nerdvegas/develop

Develop

src/rez/config.py

@@ -8,8 +8,7 @@
 from rez.exceptions import ConfigurationError
 from rez import module_root_path
 from rez.system import system
-from rez.vendor.schema.schema import Schema, SchemaError, Optional, And, Or, Use
-from rez.vendor.enum import Enum
+from rez.vendor.schema.schema import Schema, SchemaError, And, Or, Use
 from rez.vendor import yaml
 from rez.vendor.yaml.error import YAMLError
 from rez.backport.lru_cache import lru_cache
@@ -362,6 +361,7 @@ def _parse_env_var(self, value):
     "rez_1_environment_variables":                  Bool,
     "rez_1_cmake_variables":                        Bool,
     "disable_rez_1_compatibility":                  Bool,
+    "make_package_temporarily_writable":            Bool,
     "env_var_separators":                           Dict,
     "variant_select_mode":                          VariantSelectMode_,
     "package_filter":                               OptionalDictOrDictList,
@@ -491,7 +491,7 @@ def sourced_filepaths(self):
         Returns:
             List of str: The sourced files.
         """
-        _ = self._data  # force a config load
+        _ = self._data  # noqa; force a config load
         return self._sourced_filepaths
 
     @cached_property

src/rez/package_copy.py

@@ -1,15 +1,17 @@
 from functools import partial
 import os.path
+import shutil
 import time
 
 from rez.config import config
 from rez.exceptions import PackageCopyError
 from rez.package_repository import package_repository_manager
 from rez.serialise import FileFormat
+from rez.utils import with_noop
 from rez.utils.sourcecode import IncludeModuleManager
 from rez.utils.logging_ import print_info
 from rez.utils.filesystem import replacing_symlink, replacing_copy, \
-    safe_makedirs, additive_copytree
+    safe_makedirs, additive_copytree, make_path_writable, get_existing_path
 
 
 def copy_package(package, dest_repository, variants=None, shallow=False,
@@ -188,13 +190,14 @@ def finalize():
                     dest_pkg_repo=dest_pkg_repo,
                     shallow=shallow,
                     follow_symlinks=follow_symlinks,
-                    overrides=overrides
+                    overrides=overrides,
+                    verbose=verbose
                 )
 
             # construct overrides
             overrides_ = overrides.copy()
 
-            if not keep_timestamp:
+            if not keep_timestamp and "timestamp" not in overrides:
                 overrides_["timestamp"] = int(time.time())
 
             # install the variant into the package definition
@@ -213,7 +216,7 @@ def finalize():
 
 
 def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
-                          follow_symlinks=False, overrides=None):
+                          follow_symlinks=False, overrides=None, verbose=False):
         # Get payload path of source variant. For some types (eg from a "memory"
         # type repo) there may not be a root.
         #
@@ -234,16 +237,18 @@ def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
         dest_variant_version = overrides.get("version") or src_variant.version
 
         # determine variant installation path
-        variant_install_path = dest_pkg_repo.get_package_payload_path(
+        dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path(
             package_name=dest_variant_name,
             package_version=dest_variant_version
         )
 
         if src_variant.subpath:
-            variant_install_path = os.path.join(variant_install_path,
+            variant_install_path = os.path.join(dest_pkg_payload_path,
                                                 src_variant.subpath)
+        else:
+            variant_install_path = dest_pkg_payload_path
 
-        # perform the copy/symlinking
+        # get ready for copy/symlinking
         copy_func = partial(replacing_copy,
                             follow_symlinks=follow_symlinks)
 
@@ -252,29 +257,55 @@ def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
         else:
             maybe_symlink = copy_func
 
-        if src_variant.subpath:
-            # symlink/copy the last install dir to the variant root
-            safe_makedirs(os.path.dirname(variant_install_path))
-            maybe_symlink(variant_root, variant_install_path)
+        # possibly make install path temporarily writable
+        last_dir = get_existing_path(
+            variant_install_path,
+            topmost_path=os.path.dirname(dest_pkg_payload_path))
+
+        if last_dir:
+            ctxt = make_path_writable(last_dir)
         else:
+            ctxt = with_noop()
+
+        # copy the variant payload
+        with ctxt:
             safe_makedirs(variant_install_path)
 
-            # Symlink/copy all files and dirs within the null variant, except
-            # for the package definition itself.
-            #
-            for name in os.listdir(variant_root):
-                is_pkg_defn = False
+            # determine files not to copy
+            skip_files = []
 
-                # skip package definition file
-                name_ = os.path.splitext(name)[0]
-                if name_ in config.plugins.package_repository.filesystem.package_filenames:
+            if src_variant.subpath:
+                # Detect overlapped variants. This is the case where one variant subpath
+                # might be A, and another is A/B. We must ensure that A/B is not created
+                # as a symlink during shallow install of variant A - that would then
+                # cause A/B payload to be installed back into original package, possibly
+                # corrupting it.
+                #
+                # Here we detect this case, and create a list of dirs not to copy/link,
+                # because they are in fact a subpath dir for another variant.
+                #
+                skip_files.extend(_get_overlapped_variant_dirs(src_variant))
+            else:
+                # just skip package definition file
+                for name in config.plugins.package_repository.filesystem.package_filenames:
                     for fmt in (FileFormat.py, FileFormat.yaml):
-                        filename = name_ + '.' + fmt.extension
-                        if name == filename:
-                            is_pkg_defn = True
-                            break
+                        filename = name + '.' + fmt.extension
+                        skip_files.append(filename)
+
+            # copy/link all topmost files within the variant root
+            for name in os.listdir(variant_root):
+                if name in skip_files:
+                    filepath = os.path.join(variant_root, name)
+
+                    if verbose:
+                        if src_variant.subpath:
+                            msg = ("Did not copy %s - this is part of an "
+                                   "overlapping variant's root path.")
+                        else:
+                            msg = "Did not copy package definition file %s"
+
+                        print_info(msg, filepath)
 
-                if is_pkg_defn:
                     continue
 
                 src_path = os.path.join(variant_root, name)
@@ -285,6 +316,41 @@ def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
                 else:
                     maybe_symlink(src_path, dest_path)
 
+        # copy permissions of source variant dirs onto dest
+        src_package = src_variant.parent
+        src_pkg_repo = src_package.repository
+
+        src_pkg_payload_path = src_pkg_repo.get_package_payload_path(
+            package_name=src_package.name,
+            package_version=src_package.version
+        )
+
+        shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path)
+
+        subpath = src_variant.subpath
+        while subpath:
+            src_path = os.path.join(src_pkg_payload_path, subpath)
+            dest_path = os.path.join(dest_pkg_payload_path, subpath)
+            shutil.copystat(src_path, dest_path)
+            subpath = os.path.dirname(subpath)
+
+
+def _get_overlapped_variant_dirs(src_variant):
+    package = src_variant.parent
+    dirs = set()
+
+    # find other variants that overlap src_variant and have deeper subpath
+    for variant in package.iter_variants():
+        if variant.index == src_variant.index:
+            continue
+
+        if variant.root.startswith(src_variant.root + os.path.sep):
+            relpath = os.path.relpath(variant.root, src_variant.root)
+            topmost_dir = relpath.split(os.path.sep)[0]
+            dirs.add(topmost_dir)
+
+    return list(dirs)
+
 
 def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None):
     src_include_modules_path = \
@@ -304,8 +370,17 @@ def _copy_package_include_modules(src_package, dest_pkg_repo, overrides=None):
     dest_include_modules_path = \
         os.path.join(pkg_install_path, IncludeModuleManager.include_modules_subpath)
 
-    safe_makedirs(dest_include_modules_path)
-    additive_copytree(src_include_modules_path, dest_include_modules_path)
+    last_dir = get_existing_path(dest_include_modules_path,
+                                 topmost_path=os.path.dirname(pkg_install_path))
+
+    if last_dir:
+        ctxt = make_path_writable(last_dir)
+    else:
+        ctxt = with_noop()
+
+    with ctxt:
+        safe_makedirs(dest_include_modules_path)
+        additive_copytree(src_include_modules_path, dest_include_modules_path)
 
 
 # Copyright 2013-2016 Allan Johns.

src/rez/rezconfig.py

@@ -564,6 +564,14 @@
 # no editor prompt.
 prompt_release_message = False
 
+# Sometimes a studio will run a post-release process to set a package and its
+# payload to read-only. If you set this option to True, processes that mutate an
+# existing package (such as releasing a variant into an existing package, or
+# copying a package) will, if possible, temporarily make a package writable
+# during these processes. The mode will be set back to original afterwards.
+#
+make_package_temporarily_writable = True
+
 
 ###############################################################################
 # Suites

src/rez/tests/test_copy_package.py

@@ -1,6 +1,11 @@
 """
 test package copying
 """
+import shutil
+import time
+import os.path
+import os
+
 from rez.system import system
 from rez.build_process_ import create_build_process
 from rez.build_system import create_build_system
@@ -9,9 +14,6 @@
 from rez.package_copy import copy_package
 from rez.vendor.version.version import VersionRange
 from rez.tests.util import TestBase, TempdirMixin
-import shutil
-import os.path
-import os
 
 
 class TestCopyPackage(TestBase, TempdirMixin):
@@ -239,6 +241,67 @@ def test_4(self):
         self.assertEqual(dest_variant.handle, result_variant.handle)
 
     def test_5(self):
+        """Package copy with standard, new timestamp."""
+        self._reset_dest_repository()
+
+        # wait 1 second to guarantee newer timestamp in copied pkg
+        time.sleep(1)
+
+        # copy package and overwrite timestamp
+        src_pkg = self._get_src_pkg("floob", "1.2.0")
+        copy_package(
+            package=src_pkg,
+            dest_repository=self.dest_install_root
+        )
+
+        # check copied variant contains expected timestamp
+        dest_pkg = self._get_dest_pkg("floob", "1.2.0")
+        self.assertTrue(dest_pkg.timestamp > src_pkg.timestamp)
+
+    def test_6(self):
+        """Package copy with keep_timestamp."""
+        self._reset_dest_repository()
+
+        # wait 1 second to ensure we don't just accidentally get same timestamp
+        time.sleep(1)
+
+        # copy package and overwrite timestamp
+        src_pkg = self._get_src_pkg("floob", "1.2.0")
+        copy_package(
+            package=src_pkg,
+            dest_repository=self.dest_install_root,
+            keep_timestamp=True
+        )
+
+        # check copied variant contains expected timestamp
+        dest_pkg = self._get_dest_pkg("floob", "1.2.0")
+        self.assertEqual(dest_pkg.timestamp, src_pkg.timestamp)
+
+    def test_7(self):
+        """Package copy with overrides."""
+        self._reset_dest_repository()
+
+        overrides = {
+            "timestamp": 10000,
+            "description": "this is a copy",
+            "some_extra_key": True
+        }
+
+        # copy package and overwrite timestamp
+        src_pkg = self._get_src_pkg("floob", "1.2.0")
+        copy_package(
+            package=src_pkg,
+            dest_repository=self.dest_install_root,
+            overrides=overrides
+        )
+
+        # check copied variant contains expected timestamp
+        dest_pkg = self._get_dest_pkg("floob", "1.2.0")
+
+        for k, v in overrides.iteritems():
+            self.assertEqual(getattr(dest_pkg, k), v)
+
+    def test_8(self):
         """Ensure that include modules are copied."""
         self._reset_dest_repository()
 

src/rez/utils/__init__.py

@@ -1,6 +1,12 @@
+from contextlib import contextmanager
 import sys
 
 
+@contextmanager
+def with_noop():
+    yield
+
+
 def reraise(exc, new_exc_cls=None, format_str=None):
     if new_exc_cls is None:
         raise

src/rez/utils/_version.py

@@ -1,7 +1,7 @@
 
 
 # Update this value to version up Rez. Do not place anything else in this file.
-_rez_version = "2.26.4"
+_rez_version = "2.27.0"
 
 try:
     from rez.vendor.version.version import Version