Add YAML environment file support

docs/environments.rst

@@ -3,7 +3,15 @@ Environments
 ============
 
 When running stacker, you can optionally provide an "environment" file. The
-stacker config file will be interpolated as a `string.Template
+environment file defines values, which can then be referred to by name from
+your stack config file. The environment file is interpreted as YAML if it
+ends in `.yaml` or `.yml`, otherwise it's interpreted as simple key/value
+pairs.
+
+Key/Value environments
+----------------------
+
+The stacker config file will be interpolated as a `string.Template
 <https://docs.python.org/2/library/string.html#template-strings>`_ using the
 key/value pairs from the environment file. The format of the file is a single
 key/value per line, separated by a colon (**:**), like this::
@@ -43,6 +51,58 @@ files in your config. For example::
       variables:
         InstanceType: ${web_instance_type}
 
+YAML environments
+-----------------
+
+YAML environments allow for more complex environment configuration rather
+than simple text substitution, and support YAML features like anchors and
+references. To build on the example above, let's define a stack that's
+a little more complex::
+
+  stacks:
+    - name: webservers
+      class_path: stacker_blueprints.asg.AutoscalingGroup
+      variables:
+        InstanceType: ${web_instance_type}
+        IngressCIDRsByPort: ${ingress_cidrs_by_port}
+
+We've defined a stack which expects a list of ingress CIDR's allowed access to
+each port. Our environment files would look like this::
+
+  # in the file: stage.yml
+  web_instance_type: m3.medium
+  ingress_cidrs_by_port:
+    80:
+      - 192.168.1.0/8
+    8080:
+      - 0.0.0.0/0
+
+  # in the file: prod.env
+  web_instance_type: c4.xlarge
+  ingress_cidrs_by_port:
+    80:
+      - 192.168.1.0/8
+    443:
+      - 10.0.0.0/16
+      - 10.1.0.0/16
+
+The YAML format allows for specifying lists, maps, and supports all `pyyaml`
+functionality allowed in `safe_load()` function.
+
+Variable substitution in the YAML case is a bit more complex than in the
+`string.Template` case. Objects can only be substituted for variables in the
+case where we perform a full substitution, such as this::
+
+  vpcID: ${vpc_variable}
+
+We can not substitute an object in a sub-string, such as this::
+
+  vpcID: prefix-${vpc_variable}
+
+It makes no sense to substitute a complex object in this case, and we will raise
+an error if that happens. You can still perform this substitution with
+primitives; numbers, strings, but not dicts or lists.
+
 .. note::
   Namespace defined in the environment file has been deprecated in favor of
   defining the namespace in the config and will be removed in a future release.

stacker/commands/stacker/base.py

@@ -7,8 +7,13 @@
 import signal
 from collections import Mapping
 import logging
+import os.path
 
-from ...environment import parse_environment
+from ...environment import (
+    DictWithSourceType,
+    parse_environment,
+    parse_yaml_environment
+)
 
 logger = logging.getLogger(__name__)
 
@@ -63,8 +68,14 @@ def key_value_arg(string):
 
 def environment_file(input_file):
     """Reads a stacker environment file and returns the resulting data."""
+
+    is_yaml = os.path.splitext(input_file)[1].lower() in ['.yaml', '.yml']
+
     with open(input_file) as fd:
-        return parse_environment(fd.read())
+        if is_yaml:
+            return parse_yaml_environment(fd.read())
+        else:
+            return parse_environment(fd.read())
 
 
 class BaseCommand(object):
@@ -158,12 +169,17 @@ def add_arguments(self, parser):
             "-v", "--verbose", action="count", default=0,
             help="Increase output verbosity. May be specified up to twice.")
         parser.add_argument(
-            "environment", type=environment_file, nargs='?', default={},
-            help="Path to a simple `key: value` pair environment file. The "
-                 "values in the environment file can be used in the stack "
-                 "config as if it were a string.Template type: "
+            "environment", type=environment_file, nargs='?',
+            default=DictWithSourceType('simple'),
+            help="Path to an environment file. The file can be a simple "
+                 "`key: value` pair environment file, or a YAML file ending in"
+                 ".yaml or .yml. In the simple key:value case, values in the "
+                 "environment file can be used in the stack config as if it "
+                 "were a string.Template type: "
                  "https://docs.python.org/2/library/"
-                 "string.html#template-strings.")
+                 "string.html#template-strings. In the YAML case, variable"
+                 "references in the stack config are replaced with the objects"
+                 "in the environment after parsing")
         parser.add_argument(
             "config", type=argparse.FileType(),
             help="The config file where stack configuration is located. Must "

stacker/config/__init__.py

@@ -3,10 +3,12 @@
 from __future__ import absolute_import
 from future import standard_library
 standard_library.install_aliases()
+from past.types import basestring
 from builtins import str
 import copy
 import sys
 import logging
+import re
 
 from string import Template
 from io import StringIO
@@ -32,6 +34,7 @@
 from ..lookups import register_lookup_handler
 from ..util import merge_map, yaml_to_ordered_dict, SourceProcessor
 from .. import exceptions
+from ..environment import DictWithSourceType
 
 # register translators (yaml constructors)
 from .translators import *  # NOQA
@@ -83,33 +86,138 @@ def render(raw_config, environment=None):
 
     Args:
         raw_config (str): the raw stacker configuration string.
-        environment (dict, optional): any environment values that should be
-            passed to the config
+        environment (DictWithSourceType, optional): any environment values that
+            should be passed to the config
 
     Returns:
         str: the stacker configuration populated with any values passed from
             the environment
 
     """
-
-    t = Template(raw_config)
-    buff = StringIO()
     if not environment:
         environment = {}
-    try:
-        substituted = t.substitute(environment)
-    except KeyError as e:
-        raise exceptions.MissingEnvironment(e.args[0])
-    except ValueError:
-        # Support "invalid" placeholders for lookup placeholders.
-        substituted = t.safe_substitute(environment)
-
-    if not isinstance(substituted, str):
-        substituted = substituted.decode('utf-8')
-
-    buff.write(substituted)
-    buff.seek(0)
-    return buff.read()
+    # If we have a naked dict, we got here through the old non-YAML path, so
+    # we can't have a YAML config file.
+    is_yaml = False
+    if type(environment) == DictWithSourceType:
+        is_yaml = environment.source_type == 'yaml'
+
+    if is_yaml:
+        # First, read the config as yaml
+        config = yaml.safe_load(raw_config)
+
+        # Next, we need to walk the yaml structure, and find all things which
+        # look like variable references. This regular expression is copied from
+        # string.template to match variable references identically as the
+        # simple configuration case below. We've got two cases of this pattern,
+        # since python 2.7 doesn't support re.fullmatch(), so we have to add
+        # the end of line anchor to the inner patterns.
+        idpattern = r'[_a-z][_a-z0-9]*'
+        pattern = r"""
+            %(delim)s(?:
+              (?P<named>%(id)s)         |   # delimiter and a Python identifier
+              {(?P<braced>%(id)s)}         # delimiter and a braced identifier
+            )
+            """ % {'delim': re.escape('$'),
+                   'id': idpattern,
+                   }
+        full_pattern = r"""
+            %(delim)s(?:
+              (?P<named>%(id)s)$         |  # delimiter and a Python identifier
+              {(?P<braced>%(id)s)}$         # delimiter and a braced identifier
+            )
+            """ % {'delim': re.escape('$'),
+                   'id': idpattern,
+                   }
+        exp = re.compile(pattern, re.IGNORECASE | re.VERBOSE)
+        full_exp = re.compile(full_pattern, re.IGNORECASE | re.VERBOSE)
+        new_config = substitute_references(config, environment, exp, full_exp)
+        # Now, re-encode the whole thing as YAML and return that.
+        return yaml.safe_dump(new_config)
+    else:
+        t = Template(raw_config)
+        buff = StringIO()
+
+        try:
+            substituted = t.substitute(environment)
+        except KeyError as e:
+            raise exceptions.MissingEnvironment(e.args[0])
+        except ValueError:
+            # Support "invalid" placeholders for lookup placeholders.
+            substituted = t.safe_substitute(environment)
+
+        if not isinstance(substituted, str):
+            substituted = substituted.decode('utf-8')
+
+        buff.write(substituted)
+        buff.seek(0)
+        return buff.read()
+
+
+def substitute_references(root, environment, exp, full_exp):
+    # We need to check for something being a string in both python 2.7 and
+    # 3+. The aliases in the future package don't work for yaml sourced
+    # strings, so we have to spin our own.
+    def isstr(s):
+        try:
+            return isinstance(s, basestring)
+        except NameError:
+            return isinstance(s, str)
+
+    if isinstance(root, list):
+        result = []
+        for x in root:
+            result.append(substitute_references(x, environment, exp, full_exp))
+        return result
+    elif isinstance(root, dict):
+        result = {}
+        for k, v in root.items():
+            result[k] = substitute_references(v, environment, exp, full_exp)
+        return result
+    elif isstr(root):
+        # Strings are the special type where all substitutions happen. If we
+        # encounter a string object in the expression tree, we need to perform
+        # one of two different kinds of matches on it. First, if the entire
+        # string is a variable, we can replace it with an arbitrary object;
+        # dict, list, primitive. If the string contains variables within it,
+        # then we have to do string substitution.
+        match_obj = full_exp.match(root.strip())
+        if match_obj:
+            matches = match_obj.groupdict()
+            var_name = matches['named'] or matches['braced']
+            if var_name is not None:
+                value = environment.get(var_name)
+                if value is None:
+                    raise exceptions.MissingEnvironment(var_name)
+                return value
+
+        # Returns if an object is a basic type. Once again, the future package
+        # overrides don't work for string here, so we have to special case it
+        def is_basic_type(o):
+            if isstr(o):
+                return True
+            basic_types = [int, bool, float]
+            for t in basic_types:
+                if isinstance(o, t):
+                    return True
+            return False
+
+        # If we got here, then we didn't have any full matches, now perform
+        # partial substitutions within a string.
+        def replace(mo):
+            name = mo.groupdict()['braced'] or mo.groupdict()['named']
+            if not name:
+                return root[mo.start():mo.end()]
+            val = environment.get(name)
+            if val is None:
+                raise exceptions.MissingEnvironment(name)
+            if not is_basic_type(val):
+                raise exceptions.WrongEnvironmentType(name)
+            return str(val)
+        value = exp.sub(replace, root)
+        return value
+    # In all other unhandled cases, return a copy of the input
+    return copy.copy(root)
 
 
 def parse(raw_config):

stacker/environment.py

@@ -2,9 +2,27 @@
 from __future__ import division
 from __future__ import absolute_import
 
+import yaml
+
+
+class DictWithSourceType(dict):
+    """An environment dict which keeps track of its source.
+
+    Environment files may be loaded from simple key/value files, or from
+    structured YAML files, and we need to render them using a different
+    strategy based on their source. This class adds a source_type property
+    to a dict which keeps track of whether the source for the dict is
+    yaml or simple.
+    """
+    def __init__(self, source_type, *args):
+        dict.__init__(self, args)
+        if source_type not in ['yaml', 'simple']:
+            raise ValueError('source_type must be yaml or simple')
+        self.source_type = source_type
+
 
 def parse_environment(raw_environment):
-    environment = {}
+    environment = DictWithSourceType('simple')
     for line in raw_environment.split('\n'):
         line = line.strip()
         if not line:
@@ -20,3 +38,13 @@ def parse_environment(raw_environment):
 
         environment[key] = value.strip()
     return environment
+
+
+def parse_yaml_environment(raw_environment):
+    environment = DictWithSourceType('yaml')
+    parsed_env = yaml.safe_load(raw_environment)
+
+    if type(parsed_env) != dict:
+        raise ValueError('Environment must be valid YAML')
+    environment.update(parsed_env)
+    return environment

stacker/exceptions.py

@@ -158,6 +158,14 @@ def __init__(self, key, *args, **kwargs):
         super(MissingEnvironment, self).__init__(message, *args, **kwargs)
 
 
+class WrongEnvironmentType(Exception):
+
+    def __init__(self, key, *args, **kwargs):
+        self.key = key
+        message = "Environment key %s can't be merged into a string" % (key,)
+        super(WrongEnvironmentType, self).__init__(message, *args, **kwargs)
+
+
 class ImproperlyConfigured(Exception):
 
     def __init__(self, cls, error, *args, **kwargs):

stacker/hooks/iam.py

@@ -46,6 +46,7 @@ def create_ecs_service_role(provider, context, **kwargs):
             raise
 
     policy = Policy(
+        Version='2012-10-17',
         Statement=[
             Statement(
                 Effect=Allow,