Merge branch 'master' into ensure_s3_bucket_pr495

CHANGELOG.md

@@ -1,3 +1,14 @@
+## 1.1.2 (2017-11-01)
+
+This is a minor update to help deal with some of the issues between `stacker`
+and `stacker_blueprints` both having dependencies on `troposphere`. It loosens
+the dependencies, allowing stacker to work with any reasonably new version
+of troposphere (anything greater than `1.9.0`). `stacker_blueprints` will
+likely require newer versions of troposphere, as new types are introduced to
+the blueprints, but it's unlikely we'll change the `troposphere` version string
+for stacker, since it relies on only the most basic parts of the `troposphere`
+API.
+
 ## 1.1.1 (2017-10-11)
 
 This release is mostly about updating the dependencies for stacker to newer

RELEASE.md

@@ -0,0 +1,24 @@
+# Steps to release a new version
+
+## Preparing for the release
+
+- Check out a branch named for the version: `git checkout -b release-1.1.1`
+- Change version in setup.py and stacker/\_\_init\_\_.py
+- Update CHANGELOG.md with changes made since last release (see below for helpful
+  command)
+- add changed files: `git add setup.py stacker/\_\_init\_\_.py CHANGELOG.md`
+- Commit changes: `git commit -m "Release 1.1.1"`
+- Create a signed tag: `git tag --sign -m "Release 1.1.1" 1.1.1`
+- Push branch up to git: `git push -u origin release-1.1.1`
+- Open a PR for the release, ensure that tests pass
+
+## Releasing
+
+- Push tag: `git push --tags`
+- Merge PR into master, checkout master locally: `git checkout master; git pull`
+- Create PyPI release: `python setup.py sdist upload --sign`
+- Update github release page: https://github.com/remind101/stacker/releases 
+  - use the contents of the latest CHANGELOG entry for the body.
+
+# Helper to create CHANGELOG entries
+git log --reverse --pretty=format:"%s" | tail -100 | sed 's/^/- /'

docs/config.rst

@@ -143,7 +143,7 @@ See the AWS documentation for `AWS CloudFormation Service Roles`_.
 
 Remote Packages
 ---------------
-The **package_sources** top level keyword can be used to define remote git
+The **package_sources** top level keyword can be used to define remote
 sources for blueprints (e.g., retrieving ``stacker_blueprints`` on github at
 tag ``v1.0.2``).
 
@@ -162,13 +162,30 @@ The only required key for a git repository config is ``uri``, but ``branch``,
         - uri: git@github.com:contoso/foo.git
           commit: 12345678
 
-Use the ``paths`` option when subdirectories of the repo should be added to
-Stacker's ``sys.path``.
-
 If no specific commit or tag is specified for a repo, the remote repository
 will be checked for newer commits on every execution of Stacker.
 
-Cloned repositories will be cached between builds; the cache location defaults
+For ``.tar.gz`` & ``zip`` archives on s3, specify a ``bucket`` & ``key``::
+
+    package_sources:
+      s3:
+        - bucket: mystackers3bucket
+          key: archives/blueprints-v1.zip
+          paths:
+            - stacker_blueprints
+        - bucket: anothers3bucket
+          key: public/public-blueprints-v2.tar.gz
+          requester_pays: true
+        - bucket: yetanothers3bucket
+          key: sallys-blueprints-v1.tar.gz
+          # use_latest defaults to true - will update local copy if the
+          # last modified date on S3 changes
+          use_latest: false
+
+Use the ``paths`` option when subdirectories of the repo/archive should be
+added to Stacker's ``sys.path``.
+
+Cloned repos/archives will be cached between builds; the cache location defaults
 to ~/.stacker but can be manually specified via the **stacker_cache_dir** top
 level keyword.
 

setup.py

@@ -1,24 +1,25 @@
 import os
 from setuptools import setup, find_packages
 
-VERSION = "1.1.1"
+VERSION = "1.1.2"
 
 src_dir = os.path.dirname(__file__)
 
 install_requires = [
-    "troposphere~=2.0.0",
+    "troposphere>=1.9.0",
     "boto3>=1.3.1,<1.5.0",
     "PyYAML~=3.12",
-    "awacs~=0.7.1",
+    "awacs>=0.6.0",
     "colorama~=0.3.7",
     "formic~=0.9b",
     "gitpython~=2.0",
-    "schematics~=2.0.1"
+    "schematics~=2.0.1",
+    "python-dateutil~=2.0"
 ]
 
 tests_require = [
     "mock~=2.0.0",
-    "moto~=0.4.30",
+    "moto~=1.1.24",
     "testfixtures~=4.10.0",
     "coverage~=4.3.4"
 ]
@@ -48,7 +49,7 @@ def read(filename):
         author_email="loki77@gmail.com",
         license="New BSD license",
         url="https://github.com/remind101/stacker",
-        description="Opinionated AWS CloudFormation Stack manager",
+        description="AWS CloudFormation Stack manager",
         long_description=read("README.rst"),
         packages=find_packages(),
         scripts=scripts,

stacker/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.1"
+__version__ = "1.1.2"

stacker/config/__init__.py

@@ -242,9 +242,21 @@ class GitPackageSource(Model):
     configs = ListType(StringType, serialize_when_none=False)
 
 
+class S3PackageSource(Model):
+    bucket = StringType(required=True)
+
+    key = StringType(required=True)
+
+    use_latest = BooleanType(serialize_when_none=False)
+
+    requester_pays = BooleanType(serialize_when_none=False)
+
+
 class PackageSources(Model):
     git = ListType(ModelType(GitPackageSource))
 
+    s3 = ListType(ModelType(S3PackageSource))
+
 
 class Hook(Model):
     path = StringType(required=True)
@@ -357,10 +369,11 @@ def validate(self):
 
     def validate_stacks(self, data, value):
         if value:
-            names = set()
-            for i, stack in enumerate(value):
-                if stack.name in names:
-                    raise ValidationError(
-                        "Duplicate stack %s found at index %d."
-                        % (stack.name, i))
-                names.add(stack.name)
+            stack_names = [stack.name for stack in value]
+            if len(set(stack_names)) != len(stack_names):
+                # only loop / enumerate if there is an issue.
+                for i, stack_name in enumerate(stack_names):
+                    if stack_names.count(stack_name) != 1:
+                        raise ValidationError(
+                            "Duplicate stack %s found at index %d."
+                            % (stack_name, i))

stacker/lookups/handlers/envvar.py

@@ -30,5 +30,5 @@ def handler(value, **kwargs):
 
     try:
         return os.environ[value]
-    except:
+    except KeyError:
         raise ValueError('EnvVar "{}" does not exist'.format(value))

stacker/plan.py

@@ -355,6 +355,11 @@ def dump(self, directory, context, provider=None):
             blueprint = step.stack.blueprint
             filename = stack_template_key_name(blueprint)
             path = os.path.join(directory, filename)
+
+            blueprint_dir = os.path.dirname(path)
+            if not os.path.exists(blueprint_dir):
+                os.makedirs(blueprint_dir)
+
             logger.info("Writing stack \"%s\" -> %s", step_name, path)
             with open(path, "w") as f:
                 f.write(blueprint.rendered)

stacker/tests/test_config.py

@@ -14,6 +14,8 @@
 from stacker import exceptions
 from stacker.lookups.registry import LOOKUP_HANDLERS
 
+from yaml.constructor import ConstructorError
+
 config = """a: $a
 b: $b
 c: $c"""
@@ -97,7 +99,7 @@ def test_config_validate_duplicate_stack_names(self):
         error = ex.exception.errors['stacks'][0]
         self.assertEquals(
             error.__str__(),
-            "Duplicate stack bastion found at index 1.")
+            "Duplicate stack bastion found at index 0.")
 
     def test_dump_unicode(self):
         config = Config()
@@ -196,6 +198,15 @@ def test_parse(self):
             args:
               domain: mydomain.com
         package_sources:
+          s3:
+            - bucket: acmecorpbucket
+              key: public/acmecorp-blueprints-v1.zip
+            - bucket: examplecorpbucket
+              key: public/examplecorp-blueprints-v2.tar.gz
+              requester_pays: true
+            - bucket: anotherexamplebucket
+              key: example-blueprints-v3.tar.gz
+              use_latest: false
           git:
             - uri: git@github.com:acmecorp/stacker_blueprints.git
             - uri: git@github.com:remind101/stacker_blueprints.git
@@ -248,6 +259,15 @@ def test_parse(self):
             args:
               domain: mydomain.com
         package_sources:
+          s3:
+            - bucket: acmecorpbucket
+              key: public/acmecorp-blueprints-v1.zip
+            - bucket: examplecorpbucket
+              key: public/examplecorp-blueprints-v2.tar.gz
+              requester_pays: true
+            - bucket: anotherexamplebucket
+              key: example-blueprints-v3.tar.gz
+              use_latest: false
           git:
             - uri: git@github.com:acmecorp/stacker_blueprints.git
             - uri: git@github.com:remind101/stacker_blueprints.git
@@ -290,6 +310,25 @@ def test_parse(self):
                 self.assertEqual(
                     hooks[0].args, {"domain": "mydomain.com"})
 
+            self.assertEqual(
+                config.package_sources.s3[0].bucket,
+                "acmecorpbucket")
+            self.assertEqual(
+                config.package_sources.s3[0].key,
+                "public/acmecorp-blueprints-v1.zip")
+            self.assertEqual(
+                config.package_sources.s3[1].bucket,
+                "examplecorpbucket")
+            self.assertEqual(
+                config.package_sources.s3[1].key,
+                "public/examplecorp-blueprints-v2.tar.gz")
+            self.assertEqual(
+                config.package_sources.s3[1].requester_pays,
+                True)
+            self.assertEqual(
+                config.package_sources.s3[2].use_latest,
+                False)
+
             self.assertEqual(
                 config.package_sources.git[0].uri,
                 "git@github.com:acmecorp/stacker_blueprints.git")
@@ -398,6 +437,29 @@ def test_render_parse_load_namespace_fallback(self):
         config.validate()
         self.assertEquals(config.namespace, "prod")
 
+    def test_raise_constructor_error_on_duplicate_key(self):
+        yaml_config = """
+        namespace: prod
+        stacks:
+          - name: vpc
+            class_path: blueprints.VPC
+            class_path: blueprints.Fake
+        """
+        with self.assertRaises(ConstructorError):
+            parse(yaml_config)
+
+    def test_raise_construct_error_on_duplicate_stack_name(self):
+        yaml_config = """
+        namespace: prod
+        stacks:
+          my_vpc:
+            class_path: blueprints.VPC1
+          my_vpc:
+            class_path: blueprints.VPC2
+        """
+        with self.assertRaises(ConstructorError):
+            parse(yaml_config)
+
 
 if __name__ == '__main__':
     unittest.main()

stacker/tests/test_lookups.py

@@ -89,7 +89,7 @@ def test_valid_extract_lookups_from_string(self):
         _input = "vpc::PublicSubnets"
         value = "${%s %s}" % (_type, _input)
         lookups = extract_lookups_from_string(value)
-        l = lookups.pop()
-        assert l.type == _type
-        assert l.input == _input
-        assert l.raw == "%s %s" % (_type, _input)
+        lookup = lookups.pop()
+        assert lookup.type == _type
+        assert lookup.input == _input
+        assert lookup.raw == "%s %s" % (_type, _input)

stacker/tests/test_plan.py

@@ -1,7 +1,11 @@
-import unittest
+import os.path
+import shutil
+import tempfile
 
+import unittest
 import mock
 
+from stacker.actions.base import stack_template_key_name
 from stacker.context import Context
 from stacker.context import Config
 from stacker.exceptions import ImproperlyConfigured, FailedVariableLookup
@@ -325,6 +329,46 @@ def test_dump_no_provider_lookups(self, *args):
         with self.assertRaises(FailedVariableLookup):
             plan.dump("test", context=self.context)
 
+    def test_dump_create_dirs(self):
+        tmp_dir = tempfile.mkdtemp()
+        try:
+            plan = Plan(description="Test", sleep_time=0)
+            previous_stack = None
+            template_paths = []
+
+            for i in range(5):
+                overrides = {
+                    "variables": {
+                        "PublicSubnets": "1",
+                        "SshKeyName": "1",
+                        "PrivateSubnets": "1",
+                        "Random": "${noop something}",
+                    },
+                }
+                if previous_stack:
+                    overrides["requires"] = [previous_stack.fqn]
+                stack = Stack(
+                    definition=generate_definition("vpc", i, **overrides),
+                    context=self.context,
+                )
+                previous_stack = stack
+                plan.add(
+                    stack=stack,
+                    run_func=self._run_func,
+                    requires=stack.requires,
+                )
+
+                template_path = os.path.join(
+                    tmp_dir, stack_template_key_name(stack.blueprint))
+                template_paths.append(template_path)
+
+            plan.dump(tmp_dir, context=self.context)
+
+            for template_path in template_paths:
+                self.assertTrue(os.path.isfile(template_path))
+        finally:
+            shutil.rmtree(tmp_dir)
+
     def test_plan_checkpoint_interval(self):
         plan = Plan(description="Test", logger_type=BASIC_LOGGER_TYPE)
         self.assertEqual(plan.check_point_interval, 10)