change diff to use CFN change sets instead of comparing template dicts (

cloudtools#744)

* provider: add method to create, print, and delete a changeset for diff

* refactor diff command to use stack changeset rather than diff text

* provider: cleanup changeset temp stack, improve full output

* update providers/aws/test_default

- remove invalidated diff tests
- fixing aws provider tests
- add aws provider test for my changes
- linting fix

* blueprints: add method for retrieving output definitions

* add output handling for dependent stacks

* move most of logic to provider, fix issues with blueprints and rxref

* update tests for additions

* update docs

* update changelog

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Upcoming release
 
 - Ensure that base64 lookup codec encodes the bytes object as a string [GH-742]
+- Use CloudFormation Change Sets for `stacker diff`
 
 ## 1.7.0 (2019-04-07)
 

docs/commands.rst

@@ -103,7 +103,7 @@ already been destroyed).
     config                The config file where stack configuration is located.
                           Must be in yaml format. If `-` is provided, then the
                           config will be read from stdin.
-                          
+
   optional arguments:
     -h, --help            show this help message and exit
     -e ENV=VALUE, --env ENV=VALUE
@@ -182,10 +182,17 @@ config.
 Diff
 ----
 
-Diff attempts to show the differences between what stacker expects to push up
-into CloudFormation, and what already exists in CloudFormation.  This command
-is not perfect, as following things like *Ref* and *GetAtt* are not currently
-possible, but it should give a good idea if anything has changed.
+Diff creates a CloudFormation Change Set for each stack and displays the
+resulting changes. This works for stacks that already exist and new stacks.
+
+For stacks that are dependent on outputs from other stacks in the same file,
+stacker will infer that an update was made to the "parent" stack and invalidate
+outputs from resources that were changed and replace their value with
+``<inferred-change: stackName.outputName=unresolvedValue>``. This is done to
+illustrate the potential blast radius of a change and assist in tracking down
+why subsequent stacks could change. This inference is not perfect but takes a
+"best effort" approach to showing potential change between stacks that rely on
+each others outputs.
 
 ::
 

stacker/actions/diff.py

@@ -3,16 +3,12 @@
 from __future__ import absolute_import
 from builtins import str
 from builtins import object
-import difflib
-import json
 import logging
 from operator import attrgetter
 
 from .base import plan, build_walker
 from . import build
-from ..ui import ui
 from .. import exceptions
-from ..util import parse_cloudformation_template
 from ..status import (
     NotSubmittedStatus,
     NotUpdatedStatus,
@@ -148,75 +144,17 @@ def diff_parameters(old_params, new_params):
     return diff
 
 
-def normalize_json(template):
-    """Normalize our template for diffing.
-
-    Args:
-        template(str): string representing the template
-
-    Returns:
-        list: json representation of the parameters
-    """
-    obj = parse_cloudformation_template(template)
-    json_str = json.dumps(
-        obj, sort_keys=True, indent=4, default=str, separators=(',', ': '),
-    )
-    result = []
-    lines = json_str.split("\n")
-    for line in lines:
-        result.append(line + "\n")
-    return result
-
-
-def build_stack_changes(stack_name, new_stack, old_stack, new_params,
-                        old_params):
-    """Builds a list of strings to represent the the parameters (if changed)
-     and stack diff"""
-    from_file = "old_%s" % (stack_name,)
-    to_file = "new_%s" % (stack_name,)
-    lines = difflib.context_diff(
-        old_stack, new_stack,
-        fromfile=from_file, tofile=to_file,
-        n=7)  # ensure at least a few lines of context are displayed afterward
-
-    template_changes = list(lines)
-    log_lines = []
-    if not template_changes:
-        log_lines.append("*** No changes to template ***")
-    param_diffs = diff_parameters(old_params, new_params)
-    if param_diffs:
-        log_lines.append(format_params_diff(param_diffs))
-    if template_changes:
-        log_lines.append("".join(template_changes))
-    return log_lines
-
-
 class Action(build.Action):
     """ Responsible for diff'ing CF stacks in AWS and on disk
 
     Generates the build plan based on stack dependencies (these dependencies
     are determined automatically based on references to output values from
     other stacks).
 
-    The plan is then used to pull the current CloudFormation template from
-    AWS and compare it to the generated templated based on the current
-    config.
+    The plan is then used to create a changeset for a stack using a
+    generated template based on the current config.
     """
 
-    def _build_new_template(self, stack, parameters):
-        """Constructs the parameters & contents of a new stack and returns a
-        list(str) representation to be output to the user
-        """
-        log_lines = ["New template parameters:"]
-        for param in sorted(parameters,
-                            key=lambda param: param['ParameterKey']):
-            log_lines.append("%s = %s" % (param['ParameterKey'],
-                                          param['ParameterValue']))
-
-        log_lines.append("\nNew template contents:")
-        log_lines.append("".join(stack))
-        return log_lines
-
     def _diff_stack(self, stack, **kwargs):
         """Handles the diffing a stack in CloudFormation vs our config"""
         if self.cancel.wait(0):
@@ -229,51 +167,18 @@ def _diff_stack(self, stack, **kwargs):
             return NotUpdatedStatus()
 
         provider = self.build_provider(stack)
-
-        provider_stack = provider.get_stack(stack.fqn)
-
-        # get the current stack template & params from AWS
-        try:
-            [old_template, old_params] = provider.get_stack_info(
-                provider_stack)
-        except exceptions.StackDoesNotExist:
-            old_template = None
-            old_params = {}
+        tags = build.build_stack_tags(stack)
 
         stack.resolve(self.context, provider)
-        # generate our own template & params
         parameters = self.build_parameters(stack)
-        new_params = dict()
-        for p in parameters:
-            new_params[p['ParameterKey']] = p['ParameterValue']
-        new_template = stack.blueprint.rendered
-        new_stack = normalize_json(new_template)
-
-        output = ["============== Stack: %s ==============" % (stack.name,)]
-        # If this is a completely new template dump our params & stack
-        if not old_template:
-            output.extend(self._build_new_template(new_stack, parameters))
-        else:
-            # Diff our old & new stack/parameters
-            old_template = parse_cloudformation_template(old_template)
-            if isinstance(old_template, str):
-                # YAML templates returned from CFN need parsing again
-                # "AWSTemplateFormatVersion: \"2010-09-09\"\nParam..."
-                # ->
-                # AWSTemplateFormatVersion: "2010-09-09"
-                old_template = parse_cloudformation_template(old_template)
-            old_stack = normalize_json(
-                json.dumps(old_template,
-                           sort_keys=True,
-                           indent=4,
-                           default=str)
-            )
-            output.extend(build_stack_changes(stack.name, new_stack, old_stack,
-                                              new_params, old_params))
-        ui.info('\n' + '\n'.join(output))
 
-        stack.set_outputs(
-            provider.get_output_dict(provider_stack))
+        try:
+            outputs = provider.get_stack_changes(
+                stack, self._template(stack.blueprint), parameters, tags
+            )
+            stack.set_outputs(outputs)
+        except exceptions.StackDidNotChange:
+            logger.info('No changes: %s', stack.fqn)
 
         return COMPLETE
 

stacker/blueprints/base.py

@@ -339,6 +339,18 @@ def get_parameter_definitions(self):
                 output[var_name] = cfn_attrs
         return output
 
+    def get_output_definitions(self):
+        """Gets the output definitions.
+
+        Returns:
+            dict: output definitions. Keys are output names, the values
+                are dicts containing key/values for various output
+                properties.
+
+        """
+        return {k: output.to_dict() for k, output in
+                self.template.outputs.items()}
+
     def get_required_parameter_definitions(self):
         """Returns all template parameters that do not have a default value.
 

stacker/blueprints/raw.py

@@ -137,6 +137,17 @@ def get_parameter_definitions(self):
         """
         return get_template_params(self.to_dict())
 
+    def get_output_definitions(self):
+        """Gets the output definitions.
+
+        Returns:
+            dict: output definitions. Keys are output names, the values
+                are dicts containing key/values for various output
+                properties.
+
+        """
+        return self.to_dict().get('Outputs', {})
+
     def resolve_variables(self, provided_variables):
         """Resolve the values of the blueprint variables.