Merge pull request #922 from nerdvegas/issue_921-config-delay_load

added DelayLoad config primitive

src/rez/config.py

@@ -2,7 +2,7 @@
 from rez import __version__
 from rez.utils.data_utils import AttrDictWrapper, RO_AttrDictWrapper, \
     convert_dicts, cached_property, cached_class_property, LazyAttributeMeta, \
-    deep_update, ModifyList
+    deep_update, ModifyList, DelayLoad
 from rez.utils.formatting import expandvars, expanduser
 from rez.utils.logging_ import get_debug_printer
 from rez.utils.scope import scoped_format
@@ -660,10 +660,14 @@ def _swap(self, other):
         self.__dict__, other.__dict__ = other.__dict__, self.__dict__
 
     def _validate_key(self, key, value, key_schema):
+        if isinstance(value, DelayLoad):
+            value = value.get_value()
+
         if type(key_schema) is type and issubclass(key_schema, Setting):
             key_schema = key_schema(self, key)
         elif not isinstance(key_schema, Schema):
             key_schema = Schema(key_schema)
+
         return key_schema.validate(value)
 
     @cached_property
@@ -881,7 +885,8 @@ def _load_config_py(filepath):
         __file__=filepath,
 
         rez_version=__version__,
-        ModifyList=ModifyList
+        ModifyList=ModifyList,
+        DelayLoad=DelayLoad
     )
 
     g = reserved.copy()

src/rez/utils/_version.py

@@ -1,7 +1,7 @@
 
 
 # Update this value to version up Rez. Do not place anything else in this file.
-_rez_version = "2.63.0"
+_rez_version = "2.64.0"
 
 
 # Copyright 2013-2016 Allan Johns.

src/rez/utils/data_utils.py

@@ -1,6 +1,8 @@
 """
 Utilities related to managing data types.
 """
+import os.path
+
 from rez.vendor.schema.schema import Schema, Optional
 from rez.exceptions import RexError
 from threading import Lock
@@ -38,6 +40,58 @@ def apply(self, v):
         return (self.prepend or []) + v + (self.append or [])
 
 
+class DelayLoad(object):
+    """Used in config to delay load a config value from anothe file.
+
+    Supported formats:
+
+        - yaml (*.yaml, *.yml)
+        - json (*.json)
+    """
+    def __init__(self, filepath):
+        self.filepath = os.path.expanduser(filepath)
+
+    def __str__(self):
+        return "%s(%s)" % (self.__class__.__name__, self.filepath)
+
+    def get_value(self):
+        def _yaml(contents):
+            from rez.vendor import yaml
+            return yaml.load(contents, Loader=yaml.FullLoader)
+
+        def _json(contents):
+            import json
+            return json.loads(contents)
+
+        ext = os.path.splitext(self.filepath)[-1]
+        if ext in (".yaml", "yml"):
+            loader = _yaml
+        elif ext == ".json":
+            loader = _json
+        else:
+            raise ValueError(
+                "Error in DelayLoad - unsupported file format %s"
+                % self.filepath
+            )
+
+        try:
+            with open(self.filepath) as f:
+                contents = f.read()
+        except Exception as e:
+            raise ValueError(
+                "Error reading %s: %s: %s"
+                % (self, e.__class__.__name__, str(e))
+            )
+
+        try:
+            return loader(contents)
+        except Exception as e:
+            raise ValueError(
+                "Error loading from %s: %s: %s"
+                % (self, e.__class__.__name__, str(e))
+            )
+
+
 def remove_nones(**kwargs):
     """Return diict copy with nones removed.
     """

wiki/pages/_Configuring-Rez.md

@@ -78,6 +78,16 @@ The *system* object has the following attributes:
 * domain: Domain name, eg 'somestudio.com';
 * rez_version: Version of rez, eg '2.0.1'.
 
+## Delay Load
+
+It is possible to store a config setting in a separate file, which will be loaded
+only when that setting is referenced. This can be useful if you have a large value
+(such as a dict) that you don't want to pollute the main config with. YAML and
+JSON formats are supported:
+
+    # in rezconfig
+    default_relocatable_per_package = DelayLoad('/svr/configs/rez_relocs.yaml')
+
 ## Commandline Tool
 
 You can use the *rez-config* command line tool to see what the current configured settings are.