From 07ea5060a9a6bf736efa1c1a9eeb5c8df4e93e20 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Mon, 19 Nov 2018 00:42:47 +0100 Subject: [PATCH] Make lookup-logic more generic (#665) * Rewrote Lookup-parser The new parser will build the entire AST to support nested lookups. * Move dependency injection of ${output} to the lookup itself * Addressed comments * Removed dead code * Fix lint warnings * Fix lint errors after master merge * Fix lint error (unused exception) * Add warning when using old style lookups * Convert lookups to new style * Reformat code to fix linting errors --- stacker/config/__init__.py | 1 + stacker/config/translators/kms.py | 4 +- stacker/exceptions.py | 42 +- stacker/lookups/handlers/__init__.py | 34 ++ stacker/lookups/handlers/ami.py | 151 +++--- stacker/lookups/handlers/default.py | 48 +- stacker/lookups/handlers/dynamodb.py | 124 ++--- stacker/lookups/handlers/envvar.py | 41 +- stacker/lookups/handlers/file.py | 126 ++--- stacker/lookups/handlers/hook_data.py | 28 +- stacker/lookups/handlers/kms.py | 72 +-- stacker/lookups/handlers/output.py | 56 ++- stacker/lookups/handlers/rxref.py | 41 +- stacker/lookups/handlers/split.py | 44 +- stacker/lookups/handlers/ssmstore.py | 69 +-- stacker/lookups/handlers/xref.py | 35 +- stacker/lookups/registry.py | 41 +- stacker/stack.py | 28 +- stacker/tests/blueprints/test_base.py | 54 +- stacker/tests/lookups/handlers/test_ami.py | 14 +- .../tests/lookups/handlers/test_default.py | 16 +- .../tests/lookups/handlers/test_dynamodb.py | 18 +- stacker/tests/lookups/handlers/test_envvar.py | 6 +- stacker/tests/lookups/handlers/test_file.py | 22 +- .../tests/lookups/handlers/test_hook_data.py | 8 +- stacker/tests/lookups/handlers/test_kms.py | 6 +- stacker/tests/lookups/handlers/test_output.py | 5 +- stacker/tests/lookups/handlers/test_rxref.py | 7 +- stacker/tests/lookups/handlers/test_split.py | 8 +- .../tests/lookups/handlers/test_ssmstore.py | 8 +- stacker/tests/lookups/handlers/test_xref.py | 7 +- stacker/tests/lookups/test_registry.py | 38 +- stacker/tests/test_stack.py | 2 + stacker/tests/test_variables.py | 83 +--- stacker/variables.py | 463 ++++++++++++++---- 35 files changed, 1063 insertions(+), 687 deletions(-) diff --git a/stacker/config/__init__.py b/stacker/config/__init__.py index be53e2d97..96c09c359 100644 --- a/stacker/config/__init__.py +++ b/stacker/config/__init__.py @@ -234,6 +234,7 @@ def not_empty_list(value): class AnyType(BaseType): pass + class LocalPackageSource(Model): source = StringType(required=True) diff --git a/stacker/config/translators/kms.py b/stacker/config/translators/kms.py index ebe5083b1..9c2e1fe4d 100644 --- a/stacker/config/translators/kms.py +++ b/stacker/config/translators/kms.py @@ -2,9 +2,9 @@ from __future__ import division from __future__ import absolute_import # NOTE: The translator is going to be deprecated in favor of the lookup -from ...lookups.handlers.kms import handler +from ...lookups.handlers.kms import KmsLookup def kms_simple_constructor(loader, node): value = loader.construct_scalar(node) - return handler(value) + return KmsLookup.handler(value) diff --git a/stacker/exceptions.py b/stacker/exceptions.py index 1602528b6..e1ae8339f 100644 --- a/stacker/exceptions.py +++ b/stacker/exceptions.py @@ -15,17 +15,27 @@ def __init__(self, lookup, lookups, value, *args, **kwargs): message = ( "Lookup: \"{}\" has non-string return value, must be only lookup " "present (not {}) in \"{}\"" - ).format(lookup.raw, len(lookups), value) + ).format(str(lookup), len(lookups), value) super(InvalidLookupCombination, self).__init__(message, *args, **kwargs) +class InvalidLookupConcatenation(Exception): + """ + Intermediary Exception to be converted to InvalidLookupCombination once it + bubbles up there + """ + def __init__(self, lookup, lookups, *args, **kwargs): + self.lookup = lookup + self.lookups = lookups + super(InvalidLookupConcatenation, self).__init__("", *args, **kwargs) + + class UnknownLookupType(Exception): - def __init__(self, lookup, *args, **kwargs): - self.lookup = lookup - message = "Unknown lookup type: \"{}\"".format(lookup.type) + def __init__(self, lookup_type, *args, **kwargs): + message = "Unknown lookup type: \"{}\"".format(lookup_type) super(UnknownLookupType, self).__init__(message, *args, **kwargs) @@ -35,11 +45,22 @@ def __init__(self, variable_name, lookup, error, *args, **kwargs): self.lookup = lookup self.error = error message = "Couldn't resolve lookup in variable `%s`, " % variable_name - message += "lookup: ${%s}: " % lookup.raw + message += "lookup: ${%s}: " % repr(lookup) message += "(%s) %s" % (error.__class__, error) super(FailedVariableLookup, self).__init__(message, *args, **kwargs) +class FailedLookup(Exception): + """ + Intermediary Exception to be converted to FailedVariableLookup once it + bubbles up there + """ + def __init__(self, lookup, error, *args, **kwargs): + self.lookup = lookup + self.error = error + super(FailedLookup, self).__init__("Failed lookup", *args, **kwargs) + + class InvalidUserdataPlaceholder(Exception): def __init__(self, blueprint_name, exception_message, *args, **kwargs): @@ -70,6 +91,17 @@ def __init__(self, blueprint_name, variable, *args, **kwargs): super(UnresolvedVariable, self).__init__(message, *args, **kwargs) +class UnresolvedVariableValue(Exception): + """ + Intermediary Exception to be converted to UnresolvedVariable once it + bubbles up there + """ + def __init__(self, lookup, *args, **kwargs): + self.lookup = lookup + super(UnresolvedVariableValue, self).__init__( + "Unresolved lookup", *args, **kwargs) + + class MissingVariable(Exception): def __init__(self, blueprint_name, variable_name, *args, **kwargs): diff --git a/stacker/lookups/handlers/__init__.py b/stacker/lookups/handlers/__init__.py index e69de29bb..6b18bed59 100644 --- a/stacker/lookups/handlers/__init__.py +++ b/stacker/lookups/handlers/__init__.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + + +class LookupHandler(object): + @classmethod + def handle(cls, value, context, provider): + """ + Perform the actual lookup + + :param value: Parameter(s) given to this lookup + :type value: str + :param context: + :param provider: + :return: Looked-up value + :rtype: str + """ + raise NotImplementedError() + + @classmethod + def dependencies(cls, lookup_data): + """ + Calculate any dependencies required to perform this lookup. + + Note that lookup_data may not be (completely) resolved at this time. + + :param lookup_data: Parameter(s) given to this lookup + :type lookup_data VariableValue + :return: Set of stack names (str) this lookup depends on + :rtype: set + """ + del lookup_data # unused in this implementation + return set() diff --git a/stacker/lookups/handlers/ami.py b/stacker/lookups/handlers/ami.py index 1f6a0c58b..8d51c0619 100644 --- a/stacker/lookups/handlers/ami.py +++ b/stacker/lookups/handlers/ami.py @@ -5,6 +5,7 @@ import re import operator +from . import LookupHandler from ...util import read_value_from_path TYPE_NAME = "ami" @@ -19,76 +20,80 @@ def __init__(self, search_string): super(ImageNotFound, self).__init__(message) -def handler(value, provider, **kwargs): - """Fetch the most recent AMI Id using a filter - - For example: - - ${ami [@]owners:self,account,amazon name_regex:serverX-[0-9]+ architecture:x64,i386} - - The above fetches the most recent AMI where owner is self - account or amazon and the ami name matches the regex described, - the architecture will be either x64 or i386 - - You can also optionally specify the region in which to perform the AMI lookup. - - Valid arguments: - - owners (comma delimited) REQUIRED ONCE: - aws_account_id | amazon | self - - name_regex (a regex) REQUIRED ONCE: - e.g. my-ubuntu-server-[0-9]+ - - executable_users (comma delimited) OPTIONAL ONCE: - aws_account_id | amazon | self - - Any other arguments specified are sent as filters to the aws api - For example, "architecture:x86_64" will add a filter - """ # noqa - value = read_value_from_path(value) - - if "@" in value: - region, value = value.split("@", 1) - else: - region = provider.region - - ec2 = get_session(region).client('ec2') - - values = {} - describe_args = {} - - # now find any other arguments that can be filters - matches = re.findall('([0-9a-zA-z_-]+:[^\s$]+)', value) - for match in matches: - k, v = match.split(':', 1) - values[k] = v - - if not values.get('owners'): - raise Exception("'owners' value required when using ami") - owners = values.pop('owners').split(',') - describe_args["Owners"] = owners - - if not values.get('name_regex'): - raise Exception("'name_regex' value required when using ami") - name_regex = values.pop('name_regex') - - executable_users = None - if values.get('executable_users'): - executable_users = values.pop('executable_users').split(',') - describe_args["ExecutableUsers"] = executable_users - - filters = [] - for k, v in values.items(): - filters.append({"Name": k, "Values": v.split(',')}) - describe_args["Filters"] = filters - - result = ec2.describe_images(**describe_args) - - images = sorted(result['Images'], key=operator.itemgetter('CreationDate'), - reverse=True) - for image in images: - if re.match("^%s$" % name_regex, image['Name']): - return image['ImageId'] - - raise ImageNotFound(value) +class AmiLookup(LookupHandler): + @classmethod + def handle(cls, value, provider, **kwargs): + """Fetch the most recent AMI Id using a filter + + For example: + + ${ami [@]owners:self,account,amazon name_regex:serverX-[0-9]+ architecture:x64,i386} + + The above fetches the most recent AMI where owner is self + account or amazon and the ami name matches the regex described, + the architecture will be either x64 or i386 + + You can also optionally specify the region in which to perform the + AMI lookup. + + Valid arguments: + + owners (comma delimited) REQUIRED ONCE: + aws_account_id | amazon | self + + name_regex (a regex) REQUIRED ONCE: + e.g. my-ubuntu-server-[0-9]+ + + executable_users (comma delimited) OPTIONAL ONCE: + aws_account_id | amazon | self + + Any other arguments specified are sent as filters to the aws api + For example, "architecture:x86_64" will add a filter + """ # noqa + value = read_value_from_path(value) + + if "@" in value: + region, value = value.split("@", 1) + else: + region = provider.region + + ec2 = get_session(region).client('ec2') + + values = {} + describe_args = {} + + # now find any other arguments that can be filters + matches = re.findall('([0-9a-zA-z_-]+:[^\s$]+)', value) + for match in matches: + k, v = match.split(':', 1) + values[k] = v + + if not values.get('owners'): + raise Exception("'owners' value required when using ami") + owners = values.pop('owners').split(',') + describe_args["Owners"] = owners + + if not values.get('name_regex'): + raise Exception("'name_regex' value required when using ami") + name_regex = values.pop('name_regex') + + executable_users = None + if values.get('executable_users'): + executable_users = values.pop('executable_users').split(',') + describe_args["ExecutableUsers"] = executable_users + + filters = [] + for k, v in values.items(): + filters.append({"Name": k, "Values": v.split(',')}) + describe_args["Filters"] = filters + + result = ec2.describe_images(**describe_args) + + images = sorted(result['Images'], + key=operator.itemgetter('CreationDate'), + reverse=True) + for image in images: + if re.match("^%s$" % name_regex, image['Name']): + return image['ImageId'] + + raise ImageNotFound(value) diff --git a/stacker/lookups/handlers/default.py b/stacker/lookups/handlers/default.py index 860a33cc4..fc2b5c845 100644 --- a/stacker/lookups/handlers/default.py +++ b/stacker/lookups/handlers/default.py @@ -1,35 +1,41 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import + +from . import LookupHandler + + TYPE_NAME = "default" -def handler(value, **kwargs): - """Use a value from the environment or fall back to a default if the - environment doesn't contain the variable. +class DefaultLookup(LookupHandler): + @classmethod + def handle(cls, value, **kwargs): + """Use a value from the environment or fall back to a default if the + environment doesn't contain the variable. - Format of value: + Format of value: - :: + :: - For example: + For example: - Groups: ${default app_security_groups::sg-12345,sg-67890} + Groups: ${default app_security_groups::sg-12345,sg-67890} - If `app_security_groups` is defined in the environment, its defined value - will be returned. Otherwise, `sg-12345,sg-67890` will be the returned - value. + If `app_security_groups` is defined in the environment, its defined + value will be returned. Otherwise, `sg-12345,sg-67890` will be the + returned value. - This allows defaults to be set at the config file level. - """ + This allows defaults to be set at the config file level. + """ - try: - env_var_name, default_val = value.split("::", 1) - except ValueError: - raise ValueError("Invalid value for default: %s. Must be in " - ":: format." % value) + try: + env_var_name, default_val = value.split("::", 1) + except ValueError: + raise ValueError("Invalid value for default: %s. Must be in " + ":: format." % value) - if env_var_name in kwargs['context'].environment: - return kwargs['context'].environment[env_var_name] - else: - return default_val + if env_var_name in kwargs['context'].environment: + return kwargs['context'].environment[env_var_name] + else: + return default_val diff --git a/stacker/lookups/handlers/dynamodb.py b/stacker/lookups/handlers/dynamodb.py index 1789d30c6..9dcd97ce8 100644 --- a/stacker/lookups/handlers/dynamodb.py +++ b/stacker/lookups/handlers/dynamodb.py @@ -6,76 +6,80 @@ import re from stacker.session_cache import get_session +from . import LookupHandler from ...util import read_value_from_path TYPE_NAME = 'dynamodb' -def handler(value, **kwargs): - """Get a value from a dynamodb table +class DynamodbLookup(LookupHandler): + @classmethod + def handle(cls, value, **kwargs): + """Get a value from a dynamodb table - dynamodb field types should be in the following format: + dynamodb field types should be in the following format: - [:]@:.... + [:]@:.... - Note: The region is optional, and defaults to the environment's - `AWS_DEFAULT_REGION` if not specified. - """ - value = read_value_from_path(value) - table_info = None - table_keys = None - region = None - table_name = None - if '@' in value: - table_info, table_keys = value.split('@', 1) - if ':' in table_info: - region, table_name = table_info.split(':', 1) + Note: The region is optional, and defaults to the environment's + `AWS_DEFAULT_REGION` if not specified. + """ + value = read_value_from_path(value) + table_info = None + table_keys = None + region = None + table_name = None + if '@' in value: + table_info, table_keys = value.split('@', 1) + if ':' in table_info: + region, table_name = table_info.split(':', 1) + else: + table_name = table_info else: - table_name = table_info - else: - raise ValueError('Please make sure to include a tablename') - - if not table_name: - raise ValueError('Please make sure to include a dynamodb table name') - - table_lookup, table_keys = table_keys.split(':', 1) - - table_keys = table_keys.split('.') - - key_dict = _lookup_key_parse(table_keys) - new_keys = key_dict['new_keys'] - clean_table_keys = key_dict['clean_table_keys'] - - projection_expression = _build_projection_expression(clean_table_keys) - - # lookup the data from dynamodb - dynamodb = get_session(region).client('dynamodb') - try: - response = dynamodb.get_item( - TableName=table_name, - Key={ - table_lookup: new_keys[0] - }, - ProjectionExpression=projection_expression - ) - except ClientError as e: - if e.response['Error']['Code'] == 'ResourceNotFoundException': - raise ValueError( - 'Cannot find the dynamodb table: {}'.format(table_name)) - elif e.response['Error']['Code'] == 'ValidationException': - raise ValueError( - 'No dynamodb record matched the partition key: ' - '{}'.format(table_lookup)) + raise ValueError('Please make sure to include a tablename') + + if not table_name: + raise ValueError('Please make sure to include a dynamodb table ' + 'name') + + table_lookup, table_keys = table_keys.split(':', 1) + + table_keys = table_keys.split('.') + + key_dict = _lookup_key_parse(table_keys) + new_keys = key_dict['new_keys'] + clean_table_keys = key_dict['clean_table_keys'] + + projection_expression = _build_projection_expression(clean_table_keys) + + # lookup the data from dynamodb + dynamodb = get_session(region).client('dynamodb') + try: + response = dynamodb.get_item( + TableName=table_name, + Key={ + table_lookup: new_keys[0] + }, + ProjectionExpression=projection_expression + ) + except ClientError as e: + if e.response['Error']['Code'] == 'ResourceNotFoundException': + raise ValueError( + 'Cannot find the dynamodb table: {}'.format(table_name)) + elif e.response['Error']['Code'] == 'ValidationException': + raise ValueError( + 'No dynamodb record matched the partition key: ' + '{}'.format(table_lookup)) + else: + raise ValueError('The dynamodb lookup {} had an error: ' + '{}'.format(value, e)) + # find and return the key from the dynamo data returned + if 'Item' in response: + return (_get_val_from_ddb_data(response['Item'], new_keys[1:])) else: - raise ValueError('The dynamodb lookup {} had an error: ' - '{}'.format(value, e)) - # find and return the key from the dynamo data returned - if 'Item' in response: - return (_get_val_from_ddb_data(response['Item'], new_keys[1:])) - else: - raise ValueError( - 'The dynamodb record could not be found using the following ' - 'key: {}'.format(new_keys[0])) + raise ValueError( + 'The dynamodb record could not be found using the following ' + 'key: {}'.format(new_keys[0])) def _lookup_key_parse(table_keys): diff --git a/stacker/lookups/handlers/envvar.py b/stacker/lookups/handlers/envvar.py index 6551a63c2..a1d9ed5fd 100644 --- a/stacker/lookups/handlers/envvar.py +++ b/stacker/lookups/handlers/envvar.py @@ -3,35 +3,38 @@ from __future__ import absolute_import import os +from . import LookupHandler from ...util import read_value_from_path TYPE_NAME = "envvar" -def handler(value, **kwargs): - """Retrieve an environment variable. +class EnvvarLookup(LookupHandler): + @classmethod + def handle(cls, value, **kwargs): + """Retrieve an environment variable. - For example: + For example: - # In stacker we would reference the environment variable like this: - conf_key: ${envvar ENV_VAR_NAME} + # In stacker we would reference the environment variable like this: + conf_key: ${envvar ENV_VAR_NAME} - You can optionally store the value in a file, ie: + You can optionally store the value in a file, ie: - $ cat envvar_value.txt - ENV_VAR_NAME + $ cat envvar_value.txt + ENV_VAR_NAME - and reference it within stacker (NOTE: the path should be relative to - the stacker config file): + and reference it within stacker (NOTE: the path should be relative + to the stacker config file): - conf_key: ${envvar file://envvar_value.txt} + conf_key: ${envvar file://envvar_value.txt} - # Both of the above would resolve to - conf_key: ENV_VALUE - """ - value = read_value_from_path(value) + # Both of the above would resolve to + conf_key: ENV_VALUE + """ + value = read_value_from_path(value) - try: - return os.environ[value] - except KeyError: - raise ValueError('EnvVar "{}" does not exist'.format(value)) + try: + return os.environ[value] + except KeyError: + raise ValueError('EnvVar "{}" does not exist'.format(value)) diff --git a/stacker/lookups/handlers/file.py b/stacker/lookups/handlers/file.py index a57af6607..0eb87e74c 100644 --- a/stacker/lookups/handlers/file.py +++ b/stacker/lookups/handlers/file.py @@ -15,6 +15,7 @@ from troposphere import GenericHelperFn, Base64 +from . import LookupHandler from ...util import read_value_from_path @@ -23,93 +24,96 @@ _PARAMETER_PATTERN = re.compile(r'{{([::|\w]+)}}') -def handler(value, **kwargs): - """Translate a filename into the file contents. +class FileLookup(LookupHandler): + @classmethod + def handle(cls, value, **kwargs): + """Translate a filename into the file contents. - Fields should use the following format:: + Fields should use the following format:: - : + : - For example:: + For example:: - # We've written a file to /some/path: - $ echo "hello there" > /some/path + # We've written a file to /some/path: + $ echo "hello there" > /some/path - # In stacker we would reference the contents of this file with the - # following - conf_key: ${file plain:file://some/path} + # In stacker we would reference the contents of this file with the + # following + conf_key: ${file plain:file://some/path} - # The above would resolve to - conf_key: hello there + # The above would resolve to + conf_key: hello there - # Or, if we used wanted a base64 encoded copy of the file data - conf_key: ${file base64:file://some/path} + # Or, if we used wanted a base64 encoded copy of the file data + conf_key: ${file base64:file://some/path} - # The above would resolve to - conf_key: aGVsbG8gdGhlcmUK + # The above would resolve to + conf_key: aGVsbG8gdGhlcmUK - Supported codecs: + Supported codecs: - - plain + - plain - - base64 - encode the plain text file at the given path with base64 - prior to returning it + - base64 - encode the plain text file at the given path with base64 + prior to returning it - - parameterized - the same as plain, but additionally supports - referencing template parameters to create userdata that's - supplemented with information from the template, as is commonly - needed in EC2 UserData. For example, given a template parameter of - BucketName, the file could contain the following text:: + - parameterized - the same as plain, but additionally supports + referencing template parameters to create userdata that's + supplemented with information from the template, as is commonly + needed in EC2 UserData. For example, given a template parameter + of BucketName, the file could contain the following text:: - #!/bin/sh - aws s3 sync s3://{{BucketName}}/somepath /somepath + #!/bin/sh + aws s3 sync s3://{{BucketName}}/somepath /somepath - and then you could use something like this in the YAML config file:: + and then you could use something like this in the YAML config + file:: - UserData: ${file parameterized:/path/to/file} + UserData: ${file parameterized:/path/to/file} - resulting in the UserData parameter being defined as:: + resulting in the UserData parameter being defined as:: - { "Fn::Join" : ["", [ - "#!/bin/sh\\naws s3 sync s3://", - {"Ref" : "BucketName"}, - "/somepath /somepath" - ]] } + { "Fn::Join" : ["", [ + "#!/bin/sh\\naws s3 sync s3://", + {"Ref" : "BucketName"}, + "/somepath /somepath" + ]] } - - parameterized-b64 - the same as parameterized, with the results - additionally wrapped in *{ "Fn::Base64": ... }* , which is what you - actually need for EC2 UserData + - parameterized-b64 - the same as parameterized, with the results + additionally wrapped in *{ "Fn::Base64": ... }* , which is what + you actually need for EC2 UserData - When using parameterized-b64 for UserData, you should use a variable - defined as such: + When using parameterized-b64 for UserData, you should use a variable + defined as such: - .. code-block:: python + .. code-block:: python - from troposphere import AWSHelperFn + from troposphere import AWSHelperFn - "UserData": { - "type": AWSHelperFn, - "description": "Instance user data", - "default": Ref("AWS::NoValue") - } + "UserData": { + "type": AWSHelperFn, + "description": "Instance user data", + "default": Ref("AWS::NoValue") + } - and then assign UserData in a LaunchConfiguration or Instance to - *self.get_variables()["UserData"]*. Note that we use AWSHelperFn as the - type because the parameterized-b64 codec returns either a Base64 or a - GenericHelperFn troposphere object - """ + and then assign UserData in a LaunchConfiguration or Instance to + *self.get_variables()["UserData"]*. Note that we use AWSHelperFn as the + type because the parameterized-b64 codec returns either a Base64 or a + GenericHelperFn troposphere object + """ - try: - codec, path = value.split(":", 1) - except ValueError: - raise TypeError( - "File value must be of the format" - " \":\" (got %s)" % (value) - ) + try: + codec, path = value.split(":", 1) + except ValueError: + raise TypeError( + "File value must be of the format" + " \":\" (got %s)" % (value) + ) - value = read_value_from_path(path) + value = read_value_from_path(path) - return CODECS[codec](value) + return CODECS[codec](value) def _parameterize_string(raw): diff --git a/stacker/lookups/handlers/hook_data.py b/stacker/lookups/handlers/hook_data.py index 8aca70915..c27f65b93 100644 --- a/stacker/lookups/handlers/hook_data.py +++ b/stacker/lookups/handlers/hook_data.py @@ -1,20 +1,26 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import + +from . import LookupHandler + + TYPE_NAME = "hook_data" -def handler(value, context, **kwargs): - """Returns the value of a key for a given hook in hook_data. +class HookDataLookup(LookupHandler): + @classmethod + def handle(cls, value, context, **kwargs): + """Returns the value of a key for a given hook in hook_data. - Format of value: + Format of value: - :: - """ - try: - hook_name, key = value.split("::") - except ValueError: - raise ValueError("Invalid value for hook_data: %s. Must be in " - ":: format." % value) + :: + """ + try: + hook_name, key = value.split("::") + except ValueError: + raise ValueError("Invalid value for hook_data: %s. Must be in " + ":: format." % value) - return context.hook_data[hook_name][key] + return context.hook_data[hook_name][key] diff --git a/stacker/lookups/handlers/kms.py b/stacker/lookups/handlers/kms.py index b5f654d65..ba80d2779 100644 --- a/stacker/lookups/handlers/kms.py +++ b/stacker/lookups/handlers/kms.py @@ -4,60 +4,64 @@ import codecs from stacker.session_cache import get_session +from . import LookupHandler from ...util import read_value_from_path TYPE_NAME = "kms" -def handler(value, **kwargs): - """Decrypt the specified value with a master key in KMS. +class KmsLookup(LookupHandler): + @classmethod + def handle(cls, value, **kwargs): + """Decrypt the specified value with a master key in KMS. - kmssimple field types should be in the following format: + kmssimple field types should be in the following format: - [@] + [@] - Note: The region is optional, and defaults to the environment's - `AWS_DEFAULT_REGION` if not specified. + Note: The region is optional, and defaults to the environment's + `AWS_DEFAULT_REGION` if not specified. - For example: + For example: - # We use the aws cli to get the encrypted value for the string - # "PASSWORD" using the master key called "myStackerKey" in us-east-1 - $ aws --region us-east-1 kms encrypt --key-id alias/myStackerKey \ - --plaintext "PASSWORD" --output text --query CiphertextBlob + # We use the aws cli to get the encrypted value for the string + # "PASSWORD" using the master key called "myStackerKey" in + # us-east-1 + $ aws --region us-east-1 kms encrypt --key-id alias/myStackerKey \ + --plaintext "PASSWORD" --output text --query CiphertextBlob - CiD6bC8t2Y<...encrypted blob...> + CiD6bC8t2Y<...encrypted blob...> - # In stacker we would reference the encrypted value like: - conf_key: ${kms us-east-1@CiD6bC8t2Y<...encrypted blob...>} + # In stacker we would reference the encrypted value like: + conf_key: ${kms us-east-1@CiD6bC8t2Y<...encrypted blob...>} - You can optionally store the encrypted value in a file, ie: + You can optionally store the encrypted value in a file, ie: - kms_value.txt - us-east-1@CiD6bC8t2Y<...encrypted blob...> + kms_value.txt + us-east-1@CiD6bC8t2Y<...encrypted blob...> - and reference it within stacker (NOTE: the path should be relative to - the stacker config file): + and reference it within stacker (NOTE: the path should be relative + to the stacker config file): - conf_key: ${kms file://kms_value.txt} + conf_key: ${kms file://kms_value.txt} - # Both of the above would resolve to - conf_key: PASSWORD + # Both of the above would resolve to + conf_key: PASSWORD - """ - value = read_value_from_path(value) + """ + value = read_value_from_path(value) - region = None - if "@" in value: - region, value = value.split("@", 1) + region = None + if "@" in value: + region, value = value.split("@", 1) - kms = get_session(region).client('kms') + kms = get_session(region).client('kms') - # encode str value as an utf-8 bytestring for use with codecs.decode. - value = value.encode('utf-8') + # encode str value as an utf-8 bytestring for use with codecs.decode. + value = value.encode('utf-8') - # get raw but still encrypted value from base64 version. - decoded = codecs.decode(value, 'base64') + # get raw but still encrypted value from base64 version. + decoded = codecs.decode(value, 'base64') - # decrypt and return the plain text raw value. - return kms.decrypt(CiphertextBlob=decoded)["Plaintext"] + # decrypt and return the plain text raw value. + return kms.decrypt(CiphertextBlob=decoded)["Plaintext"] diff --git a/stacker/lookups/handlers/output.py b/stacker/lookups/handlers/output.py index e2fd9cea5..a40ba0fb3 100644 --- a/stacker/lookups/handlers/output.py +++ b/stacker/lookups/handlers/output.py @@ -1,32 +1,60 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import + +import re from collections import namedtuple +from . import LookupHandler + TYPE_NAME = "output" Output = namedtuple("Output", ("stack_name", "output_name")) -def handler(value, context=None, **kwargs): - """Fetch an output from the designated stack. +class OutputLookup(LookupHandler): + @classmethod + def handle(cls, value, context=None, **kwargs): + """Fetch an output from the designated stack. + + Args: + value (str): string with the following format: + ::, ie. some-stack::SomeOutput + context (:class:`stacker.context.Context`): stacker context + + Returns: + str: output from the specified stack - Args: - value (str): string with the following format: - ::, ie. some-stack::SomeOutput - context (:class:`stacker.context.Context`): stacker context + """ - Returns: - str: output from the specified stack + if context is None: + raise ValueError('Context is required') - """ + d = deconstruct(value) + stack = context.get_stack(d.stack_name) + return stack.outputs[d.output_name] - if context is None: - raise ValueError('Context is required') + @classmethod + def dependencies(cls, lookup_data): + # try to get the stack name + stack_name = '' + for data_item in lookup_data: + if not data_item.resolved(): + # We encountered an unresolved substitution. + # StackName is calculated dynamically based on context: + # e.g. ${output ${default var::source}::name} + # Stop here + return set() + stack_name = stack_name + data_item.value() + match = re.search(r'::', stack_name) + if match: + stack_name = stack_name[0:match.start()] + return {stack_name} + # else: try to append the next item - d = deconstruct(value) - stack = context.get_stack(d.stack_name) - return stack.outputs[d.output_name] + # We added all lookup_data, and still couldn't find a `::`... + # Probably an error... + return set() def deconstruct(value): diff --git a/stacker/lookups/handlers/rxref.py b/stacker/lookups/handlers/rxref.py index 6d0ecd61e..858a13a3d 100644 --- a/stacker/lookups/handlers/rxref.py +++ b/stacker/lookups/handlers/rxref.py @@ -14,31 +14,34 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import +from . import LookupHandler from .output import deconstruct TYPE_NAME = "rxref" -def handler(value, provider=None, context=None, **kwargs): - """Fetch an output from the designated stack. +class RxrefLookup(LookupHandler): + @classmethod + def handle(cls, value, provider=None, context=None, **kwargs): + """Fetch an output from the designated stack. - Args: - value (str): string with the following format: - ::, ie. some-stack::SomeOutput - provider (:class:`stacker.provider.base.BaseProvider`): subclass of the - base provider - context (:class:`stacker.context.Context`): stacker context + Args: + value (str): string with the following format: + ::, ie. some-stack::SomeOutput + provider (:class:`stacker.provider.base.BaseProvider`): subclass of + the base provider + context (:class:`stacker.context.Context`): stacker context - Returns: - str: output from the specified stack - """ + Returns: + str: output from the specified stack + """ - if provider is None: - raise ValueError('Provider is required') - if context is None: - raise ValueError('Context is required') + if provider is None: + raise ValueError('Provider is required') + if context is None: + raise ValueError('Context is required') - d = deconstruct(value) - stack_fqn = context.get_fqn(d.stack_name) - output = provider.get_output(stack_fqn, d.output_name) - return output + d = deconstruct(value) + stack_fqn = context.get_fqn(d.stack_name) + output = provider.get_output(stack_fqn, d.output_name) + return output diff --git a/stacker/lookups/handlers/split.py b/stacker/lookups/handlers/split.py index f178d1187..8908c7002 100644 --- a/stacker/lookups/handlers/split.py +++ b/stacker/lookups/handlers/split.py @@ -1,36 +1,40 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import +from . import LookupHandler TYPE_NAME = "split" -def handler(value, **kwargs): - """Split the supplied string on the given delimiter, providing a list. +class SplitLookup(LookupHandler): + @classmethod + def handle(cls, value, **kwargs): + """Split the supplied string on the given delimiter, providing a list. - Format of value: + Format of value: - :: + :: - For example: + For example: - Subnets: ${split ,::subnet-1,subnet-2,subnet-3} + Subnets: ${split ,::subnet-1,subnet-2,subnet-3} - Would result in the variable `Subnets` getting a list consisting of: + Would result in the variable `Subnets` getting a list consisting of: - ["subnet-1", "subnet-2", "subnet-3"] + ["subnet-1", "subnet-2", "subnet-3"] - This is particularly useful when getting an output from another stack that - contains a list. For example, the standard vpc blueprint outputs the list - of Subnets it creates as a pair of Outputs (PublicSubnets, PrivateSubnets) - that are comma separated, so you could use this in your config: + This is particularly useful when getting an output from another stack + that contains a list. For example, the standard vpc blueprint outputs + the list of Subnets it creates as a pair of Outputs (PublicSubnets, + PrivateSubnets) that are comma separated, so you could use this in your + config: - Subnets: ${split ,::${output vpc::PrivateSubnets}} - """ + Subnets: ${split ,::${output vpc::PrivateSubnets}} + """ - try: - delimiter, text = value.split("::", 1) - except ValueError: - raise ValueError("Invalid value for split: %s. Must be in " - ":: format." % value) + try: + delimiter, text = value.split("::", 1) + except ValueError: + raise ValueError("Invalid value for split: %s. Must be in " + ":: format." % value) - return text.split(delimiter) + return text.split(delimiter) diff --git a/stacker/lookups/handlers/ssmstore.py b/stacker/lookups/handlers/ssmstore.py index 0490eb592..2da724d30 100644 --- a/stacker/lookups/handlers/ssmstore.py +++ b/stacker/lookups/handlers/ssmstore.py @@ -5,55 +5,58 @@ from stacker.session_cache import get_session +from . import LookupHandler from ...util import read_value_from_path TYPE_NAME = "ssmstore" -def handler(value, **kwargs): - """Retrieve (and decrypt if applicable) a parameter from - AWS SSM Parameter Store. +class SsmstoreLookup(LookupHandler): + @classmethod + def handle(cls, value, **kwargs): + """Retrieve (and decrypt if applicable) a parameter from + AWS SSM Parameter Store. - ssmstore field types should be in the following format: + ssmstore field types should be in the following format: - [@]ssmkey + [@]ssmkey - Note: The region is optional, and defaults to us-east-1 if not given. + Note: The region is optional, and defaults to us-east-1 if not given. - For example: + For example: - # In stacker we would reference the encrypted value like: - conf_key: ${ssmstore us-east-1@ssmkey} + # In stacker we would reference the encrypted value like: + conf_key: ${ssmstore us-east-1@ssmkey} - You can optionally store the value in a file, ie: + You can optionally store the value in a file, ie: - ssmstore_value.txt - us-east-1@ssmkey + ssmstore_value.txt + us-east-1@ssmkey - and reference it within stacker (NOTE: the path should be relative to - the stacker config file): + and reference it within stacker (NOTE: the path should be relative + to the stacker config file): - conf_key: ${ssmstore file://ssmstore_value.txt} + conf_key: ${ssmstore file://ssmstore_value.txt} - # Both of the above would resolve to - conf_key: PASSWORD + # Both of the above would resolve to + conf_key: PASSWORD - """ - value = read_value_from_path(value) + """ + value = read_value_from_path(value) - region = "us-east-1" - if "@" in value: - region, value = value.split("@", 1) + region = "us-east-1" + if "@" in value: + region, value = value.split("@", 1) - client = get_session(region).client("ssm") - response = client.get_parameters( - Names=[ - value, - ], - WithDecryption=True - ) - if 'Parameters' in response: - return str(response['Parameters'][0]['Value']) + client = get_session(region).client("ssm") + response = client.get_parameters( + Names=[ + value, + ], + WithDecryption=True + ) + if 'Parameters' in response: + return str(response['Parameters'][0]['Value']) - raise ValueError('SSMKey "{}" does not exist in region {}'.format(value, - region)) + raise ValueError('SSMKey "{}" does not exist in region {}'.format( + value, region)) diff --git a/stacker/lookups/handlers/xref.py b/stacker/lookups/handlers/xref.py index 44c9bd30b..a318d252b 100644 --- a/stacker/lookups/handlers/xref.py +++ b/stacker/lookups/handlers/xref.py @@ -13,28 +13,31 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import +from . import LookupHandler from .output import deconstruct TYPE_NAME = "xref" -def handler(value, provider=None, **kwargs): - """Fetch an output from the designated stack. +class XrefLookup(LookupHandler): + @classmethod + def handle(cls, value, provider=None, **kwargs): + """Fetch an output from the designated stack. - Args: - value (str): string with the following format: - ::, ie. some-stack::SomeOutput - provider (:class:`stacker.provider.base.BaseProvider`): subclass of the - base provider + Args: + value (str): string with the following format: + ::, ie. some-stack::SomeOutput + provider (:class:`stacker.provider.base.BaseProvider`): subclass of + the base provider - Returns: - str: output from the specified stack - """ + Returns: + str: output from the specified stack + """ - if provider is None: - raise ValueError('Provider is required') + if provider is None: + raise ValueError('Provider is required') - d = deconstruct(value) - stack_fqn = d.stack_name - output = provider.get_output(stack_fqn, d.output_name) - return output + d = deconstruct(value) + stack_fqn = d.stack_name + output = provider.get_output(stack_fqn, d.output_name) + return output diff --git a/stacker/lookups/registry.py b/stacker/lookups/registry.py index 988c2363d..7d0fab46d 100644 --- a/stacker/lookups/registry.py +++ b/stacker/lookups/registry.py @@ -1,6 +1,10 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import + +import logging +import warnings + from past.builtins import basestring from ..exceptions import UnknownLookupType, FailedVariableLookup @@ -34,6 +38,19 @@ def register_lookup_handler(lookup_type, handler_or_path): if isinstance(handler_or_path, basestring): handler = load_object_from_string(handler_or_path) LOOKUP_HANDLERS[lookup_type] = handler + if type(handler) != type: + # Hander is a not a new-style handler + logger = logging.getLogger(__name__) + logger.warning("Registering lookup `%s`: Please upgrade to use the " + "new style of Lookups." % lookup_type) + warnings.warn( + # For some reason, this does not show up... + # Leaving it in anyway + "Lookup `%s`: Please upgrade to use the new style of Lookups" + "." % lookup_type, + DeprecationWarning, + stacklevel=2, + ) def unregister_lookup_handler(lookup_type): @@ -80,15 +97,15 @@ def resolve_lookups(variable, context, provider): return resolved_lookups -register_lookup_handler(output.TYPE_NAME, output.handler) -register_lookup_handler(kms.TYPE_NAME, kms.handler) -register_lookup_handler(ssmstore.TYPE_NAME, ssmstore.handler) -register_lookup_handler(envvar.TYPE_NAME, envvar.handler) -register_lookup_handler(xref.TYPE_NAME, xref.handler) -register_lookup_handler(rxref.TYPE_NAME, rxref.handler) -register_lookup_handler(ami.TYPE_NAME, ami.handler) -register_lookup_handler(file_handler.TYPE_NAME, file_handler.handler) -register_lookup_handler(split.TYPE_NAME, split.handler) -register_lookup_handler(default.TYPE_NAME, default.handler) -register_lookup_handler(hook_data.TYPE_NAME, hook_data.handler) -register_lookup_handler(dynamodb.TYPE_NAME, dynamodb.handler) +register_lookup_handler(output.TYPE_NAME, output.OutputLookup) +register_lookup_handler(kms.TYPE_NAME, kms.KmsLookup) +register_lookup_handler(ssmstore.TYPE_NAME, ssmstore.SsmstoreLookup) +register_lookup_handler(envvar.TYPE_NAME, envvar.EnvvarLookup) +register_lookup_handler(xref.TYPE_NAME, xref.XrefLookup) +register_lookup_handler(rxref.TYPE_NAME, rxref.RxrefLookup) +register_lookup_handler(ami.TYPE_NAME, ami.AmiLookup) +register_lookup_handler(file_handler.TYPE_NAME, file_handler.FileLookup) +register_lookup_handler(split.TYPE_NAME, split.SplitLookup) +register_lookup_handler(default.TYPE_NAME, default.DefaultLookup) +register_lookup_handler(hook_data.TYPE_NAME, hook_data.HookDataLookup) +register_lookup_handler(dynamodb.TYPE_NAME, dynamodb.DynamodbLookup) diff --git a/stacker/stack.py b/stacker/stack.py index ffdeeba1f..c0c877419 100644 --- a/stacker/stack.py +++ b/stacker/stack.py @@ -9,13 +9,8 @@ Variable, resolve_variables, ) -from .lookups.handlers.output import ( - TYPE_NAME as OUTPUT_LOOKUP_TYPE_NAME, - deconstruct, -) from .blueprints.raw import RawTemplateBlueprint -from .exceptions import FailedVariableLookup def _gather_variables(stack_def): @@ -99,22 +94,13 @@ def requires(self): # Add any dependencies based on output lookups for variable in self.variables: - for lookup in variable.lookups: - if lookup.type == OUTPUT_LOOKUP_TYPE_NAME: - - try: - d = deconstruct(lookup.input) - except ValueError as e: - raise FailedVariableLookup(self.name, lookup, e) - - if d.stack_name == self.name: - message = ( - "Variable %s in stack %s has a ciruclar reference " - "within lookup: %s" - ) % (variable.name, self.name, lookup.raw) - raise ValueError(message) - requires.add(d.stack_name) - + deps = variable.dependencies() + if self.name in deps: + message = ( + "Variable %s in stack %s has a ciruclar reference" + ) % (variable.name, self.name) + raise ValueError(message) + requires.update(deps) return requires @property diff --git a/stacker/tests/blueprints/test_base.py b/stacker/tests/blueprints/test_base.py index 141be10c7..52187aaa6 100644 --- a/stacker/tests/blueprints/test_base.py +++ b/stacker/tests/blueprints/test_base.py @@ -41,7 +41,7 @@ from stacker.variables import Variable from stacker.lookups import register_lookup_handler -from ..factories import mock_lookup, mock_context +from ..factories import mock_context def mock_lookup_handler(value, provider=None, context=None, fqn=False, @@ -424,11 +424,8 @@ class TestBlueprint(Blueprint): Variable("Param2", "${output other-stack::Output}"), Variable("Param3", 3), ] - resolved_lookups = { - mock_lookup("other-stack::Output", "output"): "Test Output", - } - for var in variables: - var.replace(resolved_lookups) + + variables[1]._value._resolve("Test Output") blueprint.resolve_variables(variables) self.assertEqual(blueprint.resolved_variables["Param1"], 1) @@ -441,15 +438,14 @@ class TestBlueprint(Blueprint): "Param1": {"type": list}, } + def return_list_something(*_args, **_kwargs): + return ["something"] + + register_lookup_handler("custom", return_list_something) blueprint = TestBlueprint(name="test", context=MagicMock()) variables = [Variable("Param1", "${custom non-string-return-val}")] - lookup = mock_lookup("non-string-return-val", "custom", - "custom non-string-return-val") - resolved_lookups = { - lookup: ["something"], - } for var in variables: - var.replace(resolved_lookups) + var._value.resolve({}, {}) blueprint.resolve_variables(variables) self.assertEqual(blueprint.resolved_variables["Param1"], ["something"]) @@ -460,15 +456,14 @@ class TestBlueprint(Blueprint): "Param1": {"type": Base64}, } + def return_obj(*_args, **_kwargs): + return Base64("test") + + register_lookup_handler("custom", return_obj) blueprint = TestBlueprint(name="test", context=MagicMock()) variables = [Variable("Param1", "${custom non-string-return-val}")] - lookup = mock_lookup("non-string-return-val", "custom", - "custom non-string-return-val") - resolved_lookups = { - lookup: Base64("test"), - } for var in variables: - var.replace(resolved_lookups) + var._value.resolve({}, {}) blueprint.resolve_variables(variables) self.assertEqual(blueprint.resolved_variables["Param1"].data, @@ -480,20 +475,17 @@ class TestBlueprint(Blueprint): "Param1": {"type": list}, } - variables = [ - Variable( - "Param1", - "${custom non-string-return-val},${output some-stack::Output}", - ) - ] - lookup = mock_lookup("non-string-return-val", "custom", - "custom non-string-return-val") - resolved_lookups = { - lookup: ["something"], - } + def return_list_something(*_args, **_kwargs): + return ["something"] + + register_lookup_handler("custom", return_list_something) + variable = Variable( + "Param1", + "${custom non-string-return-val},${output some-stack::Output}", + ) + variable._value[0].resolve({}, {}) with self.assertRaises(InvalidLookupCombination): - for var in variables: - var.replace(resolved_lookups) + variable.value() def test_get_variables(self): class TestBlueprint(Blueprint): diff --git a/stacker/tests/lookups/handlers/test_ami.py b/stacker/tests/lookups/handlers/test_ami.py index f0f8e770e..0e34b7b47 100644 --- a/stacker/tests/lookups/handlers/test_ami.py +++ b/stacker/tests/lookups/handlers/test_ami.py @@ -4,7 +4,7 @@ import unittest import mock from botocore.stub import Stubber -from stacker.lookups.handlers.ami import handler, ImageNotFound +from stacker.lookups.handlers.ami import AmiLookup, ImageNotFound import boto3 from stacker.tests.factories import SessionStub, mock_provider @@ -40,7 +40,7 @@ def test_basic_lookup_single_image(self, mock_client): ) with self.stubber: - value = handler( + value = AmiLookup.handle( value="owners:self name_regex:Fake\sImage\s\d", provider=self.provider ) @@ -68,7 +68,7 @@ def test_basic_lookup_with_region(self, mock_client): ) with self.stubber: - value = handler( + value = AmiLookup.handle( value="us-west-1@owners:self name_regex:Fake\sImage\s\d", provider=self.provider ) @@ -105,7 +105,7 @@ def test_basic_lookup_multiple_images(self, mock_client): ) with self.stubber: - value = handler( + value = AmiLookup.handle( value="owners:self name_regex:Fake\sImage\s\d", provider=self.provider ) @@ -142,7 +142,7 @@ def test_basic_lookup_multiple_images_name_match(self, mock_client): ) with self.stubber: - value = handler( + value = AmiLookup.handle( value="owners:self name_regex:Fake\sImage\s\d", provider=self.provider ) @@ -160,7 +160,7 @@ def test_basic_lookup_no_matching_images(self, mock_client): with self.stubber: with self.assertRaises(ImageNotFound): - handler( + AmiLookup.handle( value="owners:self name_regex:Fake\sImage\s\d", provider=self.provider ) @@ -188,7 +188,7 @@ def test_basic_lookup_no_matching_images_from_name(self, mock_client): with self.stubber: with self.assertRaises(ImageNotFound): - handler( + AmiLookup.handle( value="owners:self name_regex:MyImage\s\d", provider=self.provider ) diff --git a/stacker/tests/lookups/handlers/test_default.py b/stacker/tests/lookups/handlers/test_default.py index 5bca2df5b..a59ccd6d8 100644 --- a/stacker/tests/lookups/handlers/test_default.py +++ b/stacker/tests/lookups/handlers/test_default.py @@ -5,7 +5,7 @@ import unittest from stacker.context import Context -from stacker.lookups.handlers.default import handler +from stacker.lookups.handlers.default import DefaultLookup class TestDefaultLookup(unittest.TestCase): @@ -20,19 +20,19 @@ def setUp(self): def test_env_var_present(self): lookup_val = "env_var::fallback" - value = handler(lookup_val, - provider=self.provider, - context=self.context) + value = DefaultLookup.handle(lookup_val, + provider=self.provider, + context=self.context) assert value == 'val_in_env' def test_env_var_missing(self): lookup_val = "bad_env_var::fallback" - value = handler(lookup_val, - provider=self.provider, - context=self.context) + value = DefaultLookup.handle(lookup_val, + provider=self.provider, + context=self.context) assert value == 'fallback' def test_invalid_value(self): value = "env_var:fallback" with self.assertRaises(ValueError): - handler(value) + DefaultLookup.handle(value) diff --git a/stacker/tests/lookups/handlers/test_dynamodb.py b/stacker/tests/lookups/handlers/test_dynamodb.py index 6cda188ce..44b6cc693 100644 --- a/stacker/tests/lookups/handlers/test_dynamodb.py +++ b/stacker/tests/lookups/handlers/test_dynamodb.py @@ -4,7 +4,7 @@ import unittest import mock from botocore.stub import Stubber -from stacker.lookups.handlers.dynamodb import handler +from stacker.lookups.handlers.dynamodb import DynamodbLookup import boto3 from stacker.tests.factories import SessionStub @@ -37,7 +37,7 @@ def test_dynamodb_handler(self, mock_client): self.get_parameters_response, expected_params) with self.stubber: - value = handler(base_lookup_key) + value = DynamodbLookup.handle(base_lookup_key) self.assertEqual(value, base_lookup_key_valid) @mock.patch('stacker.lookups.handlers.dynamodb.get_session', @@ -57,7 +57,7 @@ def test_dynamodb_number_handler(self, mock_client): self.get_parameters_response, expected_params) with self.stubber: - value = handler(base_lookup_key) + value = DynamodbLookup.handle(base_lookup_key) self.assertEqual(value, base_lookup_key_valid) @mock.patch('stacker.lookups.handlers.dynamodb.get_session', @@ -77,7 +77,7 @@ def test_dynamodb_list_handler(self, mock_client): self.get_parameters_response, expected_params) with self.stubber: - value = handler(base_lookup_key) + value = DynamodbLookup.handle(base_lookup_key) self.assertEqual(value, base_lookup_key_valid) @mock.patch('stacker.lookups.handlers.dynamodb.get_session', @@ -96,7 +96,7 @@ def test_dynamodb_empty_table_handler(self, mock_client): expected_params) with self.stubber: try: - handler(base_lookup_key) + DynamodbLookup.handle(base_lookup_key) except ValueError as e: self.assertEqual( 'Please make sure to include a dynamodb table name', @@ -117,7 +117,7 @@ def test_dynamodb_missing_table_handler(self, mock_client): expected_params) with self.stubber: try: - handler(base_lookup_key) + DynamodbLookup.handle(base_lookup_key) except ValueError as e: self.assertEqual( 'Please make sure to include a tablename', @@ -140,7 +140,7 @@ def test_dynamodb_invalid_table_handler(self, mock_client): expected_params=expected_params) with self.stubber: try: - handler(base_lookup_key) + DynamodbLookup.handle(base_lookup_key) except ValueError as e: self.assertEqual( 'Cannot find the dynamodb table: FakeTable', @@ -164,7 +164,7 @@ def test_dynamodb_invalid_partition_key_handler(self, mock_client): with self.stubber: try: - handler(base_lookup_key) + DynamodbLookup.handle(base_lookup_key) except ValueError as e: self.assertEqual( 'No dynamodb record matched the partition key: FakeKey', @@ -187,7 +187,7 @@ def test_dynamodb_invalid_partition_val_handler(self, mock_client): expected_params) with self.stubber: try: - handler(base_lookup_key) + DynamodbLookup.handle(base_lookup_key) except ValueError as e: self.assertEqual( 'The dynamodb record could not be found using ' diff --git a/stacker/tests/lookups/handlers/test_envvar.py b/stacker/tests/lookups/handlers/test_envvar.py index c3aba7022..71c9bf8a5 100644 --- a/stacker/tests/lookups/handlers/test_envvar.py +++ b/stacker/tests/lookups/handlers/test_envvar.py @@ -2,7 +2,7 @@ from __future__ import division from __future__ import absolute_import import unittest -from stacker.lookups.handlers.envvar import handler +from stacker.lookups.handlers.envvar import EnvvarLookup import os @@ -15,9 +15,9 @@ def setUp(self): os.environ[self.testkey] = self.testval def test_valid_envvar(self): - value = handler(self.testkey) + value = EnvvarLookup.handle(self.testkey) self.assertEqual(value, self.testval) def test_invalid_envvar(self): with self.assertRaises(ValueError): - handler(self.invalidtestkey) + EnvvarLookup.handle(self.invalidtestkey) diff --git a/stacker/tests/lookups/handlers/test_file.py b/stacker/tests/lookups/handlers/test_file.py index 312f71ab8..5fb27b809 100644 --- a/stacker/tests/lookups/handlers/test_file.py +++ b/stacker/tests/lookups/handlers/test_file.py @@ -11,7 +11,7 @@ import json from troposphere import Base64, GenericHelperFn, Join -from stacker.lookups.handlers.file import (json_codec, handler, +from stacker.lookups.handlers.file import (json_codec, FileLookup, parameterized_codec, yaml_codec) @@ -113,13 +113,13 @@ def test_json_codec_parameterized(self): @mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value='') def test_file_loaded(self, content_mock): - handler(u'plain:file://tmp/test') + FileLookup.handle(u'plain:file://tmp/test') content_mock.assert_called_with(u'file://tmp/test') @mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=u'Hello, world') def test_handler_plain(self, _): - out = handler(u'plain:file://tmp/test') + out = FileLookup.handle(u'plain:file://tmp/test') self.assertEqual(u'Hello, world', out) @mock.patch('stacker.lookups.handlers.file.read_value_from_path') @@ -128,7 +128,7 @@ def test_handler_b64(self, content_mock): encoded = base64.b64encode(plain.encode('utf8')) content_mock.return_value = plain - out = handler(u'base64:file://tmp/test') + out = FileLookup.handle(u'base64:file://tmp/test') self.assertEqual(encoded, out) @mock.patch('stacker.lookups.handlers.file.parameterized_codec') @@ -137,7 +137,7 @@ def test_handler_parameterized(self, content_mock, codec_mock): result = mock.Mock() codec_mock.return_value = result - out = handler(u'parameterized:file://tmp/test') + out = FileLookup.handle(u'parameterized:file://tmp/test') codec_mock.assert_called_once_with(content_mock.return_value, False) self.assertEqual(result, out) @@ -148,7 +148,7 @@ def test_handler_parameterized_b64(self, content_mock, codec_mock): result = mock.Mock() codec_mock.return_value = result - out = handler(u'parameterized-b64:file://tmp/test') + out = FileLookup.handle(u'parameterized-b64:file://tmp/test') codec_mock.assert_called_once_with(content_mock.return_value, True) self.assertEqual(result, out) @@ -159,7 +159,7 @@ def test_handler_yaml(self, content_mock, codec_mock): result = mock.Mock() codec_mock.return_value = result - out = handler(u'yaml:file://tmp/test') + out = FileLookup.handle(u'yaml:file://tmp/test') codec_mock.assert_called_once_with(content_mock.return_value, parameterized=False) @@ -171,7 +171,7 @@ def test_handler_yaml_parameterized(self, content_mock, codec_mock): result = mock.Mock() codec_mock.return_value = result - out = handler(u'yaml-parameterized:file://tmp/test') + out = FileLookup.handle(u'yaml-parameterized:file://tmp/test') codec_mock.assert_called_once_with(content_mock.return_value, parameterized=True) @@ -183,7 +183,7 @@ def test_handler_json(self, content_mock, codec_mock): result = mock.Mock() codec_mock.return_value = result - out = handler(u'json:file://tmp/test') + out = FileLookup.handle(u'json:file://tmp/test') codec_mock.assert_called_once_with(content_mock.return_value, parameterized=False) @@ -195,7 +195,7 @@ def test_handler_json_parameterized(self, content_mock, codec_mock): result = mock.Mock() codec_mock.return_value = result - out = handler(u'json-parameterized:file://tmp/test') + out = FileLookup.handle(u'json-parameterized:file://tmp/test') codec_mock.assert_called_once_with(content_mock.return_value, parameterized=True) @@ -204,4 +204,4 @@ def test_handler_json_parameterized(self, content_mock, codec_mock): @mock.patch('stacker.lookups.handlers.file.read_value_from_path') def test_unknown_codec(self, _): with self.assertRaises(KeyError): - handler(u'bad:file://tmp/test') + FileLookup.handle(u'bad:file://tmp/test') diff --git a/stacker/tests/lookups/handlers/test_hook_data.py b/stacker/tests/lookups/handlers/test_hook_data.py index e9978fed5..6dc0014d1 100644 --- a/stacker/tests/lookups/handlers/test_hook_data.py +++ b/stacker/tests/lookups/handlers/test_hook_data.py @@ -5,7 +5,7 @@ from stacker.context import Context -from stacker.lookups.handlers.hook_data import handler +from stacker.lookups.handlers.hook_data import HookDataLookup class TestHookDataLookup(unittest.TestCase): @@ -15,13 +15,13 @@ def setUp(self): self.ctx.set_hook_data("fake_hook", {"result": "good"}) def test_valid_hook_data(self): - value = handler("fake_hook::result", context=self.ctx) + value = HookDataLookup.handle("fake_hook::result", context=self.ctx) self.assertEqual(value, "good") def test_invalid_hook_data(self): with self.assertRaises(KeyError): - handler("fake_hook::bad_key", context=self.ctx) + HookDataLookup.handle("fake_hook::bad_key", context=self.ctx) def test_bad_value_hook_data(self): with self.assertRaises(ValueError): - handler("fake_hook", context=self.ctx) + HookDataLookup.handle("fake_hook", context=self.ctx) diff --git a/stacker/tests/lookups/handlers/test_kms.py b/stacker/tests/lookups/handlers/test_kms.py index 955bff0e9..bb199a639 100644 --- a/stacker/tests/lookups/handlers/test_kms.py +++ b/stacker/tests/lookups/handlers/test_kms.py @@ -8,7 +8,7 @@ import boto3 -from stacker.lookups.handlers.kms import handler +from stacker.lookups.handlers.kms import KmsLookup class TestKMSHandler(unittest.TestCase): @@ -25,12 +25,12 @@ def setUp(self): def test_kms_handler(self): with mock_kms(): - decrypted = handler(self.secret) + decrypted = KmsLookup.handle(self.secret) self.assertEqual(decrypted, self.plain) def test_kms_handler_with_region(self): region = "us-east-1" value = "%s@%s" % (region, self.secret) with mock_kms(): - decrypted = handler(value) + decrypted = KmsLookup.handle(value) self.assertEqual(decrypted, self.plain) diff --git a/stacker/tests/lookups/handlers/test_output.py b/stacker/tests/lookups/handlers/test_output.py index cd6903eb4..3891dfe25 100644 --- a/stacker/tests/lookups/handlers/test_output.py +++ b/stacker/tests/lookups/handlers/test_output.py @@ -6,7 +6,7 @@ from stacker.stack import Stack from ...factories import generate_definition -from stacker.lookups.handlers.output import handler +from stacker.lookups.handlers.output import OutputLookup class TestOutputHandler(unittest.TestCase): @@ -21,7 +21,8 @@ def test_output_handler(self): stack.set_outputs({ "SomeOutput": "Test Output"}) self.context.get_stack.return_value = stack - value = handler("stack-name::SomeOutput", context=self.context) + value = OutputLookup.handle("stack-name::SomeOutput", + context=self.context) self.assertEqual(value, "Test Output") self.assertEqual(self.context.get_stack.call_count, 1) args = self.context.get_stack.call_args diff --git a/stacker/tests/lookups/handlers/test_rxref.py b/stacker/tests/lookups/handlers/test_rxref.py index 7d29f6526..b5e7cb828 100644 --- a/stacker/tests/lookups/handlers/test_rxref.py +++ b/stacker/tests/lookups/handlers/test_rxref.py @@ -4,7 +4,7 @@ from mock import MagicMock import unittest -from stacker.lookups.handlers.rxref import handler +from stacker.lookups.handlers.rxref import RxrefLookup from ....context import Context from ....config import Config @@ -20,8 +20,9 @@ def setUp(self): def test_rxref_handler(self): self.provider.get_output.return_value = "Test Output" - value = handler("fully-qualified-stack-name::SomeOutput", - provider=self.provider, context=self.context) + value = RxrefLookup.handle("fully-qualified-stack-name::SomeOutput", + provider=self.provider, + context=self.context) self.assertEqual(value, "Test Output") args = self.provider.get_output.call_args diff --git a/stacker/tests/lookups/handlers/test_split.py b/stacker/tests/lookups/handlers/test_split.py index cdc847ce9..990799bb2 100644 --- a/stacker/tests/lookups/handlers/test_split.py +++ b/stacker/tests/lookups/handlers/test_split.py @@ -3,21 +3,21 @@ from __future__ import absolute_import import unittest -from stacker.lookups.handlers.split import handler +from stacker.lookups.handlers.split import SplitLookup class TestSplitLookup(unittest.TestCase): def test_single_character_split(self): value = ",::a,b,c" expected = ["a", "b", "c"] - assert handler(value) == expected + assert SplitLookup.handle(value) == expected def test_multi_character_split(self): value = ",,::a,,b,c" expected = ["a", "b,c"] - assert handler(value) == expected + assert SplitLookup.handle(value) == expected def test_invalid_value_split(self): value = ",:a,b,c" with self.assertRaises(ValueError): - handler(value) + SplitLookup.handle(value) diff --git a/stacker/tests/lookups/handlers/test_ssmstore.py b/stacker/tests/lookups/handlers/test_ssmstore.py index 020f79772..daff2444d 100644 --- a/stacker/tests/lookups/handlers/test_ssmstore.py +++ b/stacker/tests/lookups/handlers/test_ssmstore.py @@ -5,7 +5,7 @@ import unittest import mock from botocore.stub import Stubber -from stacker.lookups.handlers.ssmstore import handler +from stacker.lookups.handlers.ssmstore import SsmstoreLookup import boto3 from stacker.tests.factories import SessionStub @@ -46,7 +46,7 @@ def test_ssmstore_handler(self, mock_client): self.get_parameters_response, self.expected_params) with self.stubber: - value = handler(self.ssmkey) + value = SsmstoreLookup.handle(self.ssmkey) self.assertEqual(value, self.ssmvalue) self.assertIsInstance(value, str) @@ -58,7 +58,7 @@ def test_ssmstore_invalid_value_handler(self, mock_client): self.expected_params) with self.stubber: try: - handler(self.ssmkey) + SsmstoreLookup.handle(self.ssmkey) except ValueError: assert True @@ -71,5 +71,5 @@ def test_ssmstore_handler_with_region(self, mock_client): region = "us-east-1" temp_value = "%s@%s" % (region, self.ssmkey) with self.stubber: - value = handler(temp_value) + value = SsmstoreLookup.handle(temp_value) self.assertEqual(value, self.ssmvalue) diff --git a/stacker/tests/lookups/handlers/test_xref.py b/stacker/tests/lookups/handlers/test_xref.py index 91cfadacf..cb611ed65 100644 --- a/stacker/tests/lookups/handlers/test_xref.py +++ b/stacker/tests/lookups/handlers/test_xref.py @@ -4,7 +4,7 @@ from mock import MagicMock import unittest -from stacker.lookups.handlers.xref import handler +from stacker.lookups.handlers.xref import XrefLookup class TestXrefHandler(unittest.TestCase): @@ -15,8 +15,9 @@ def setUp(self): def test_xref_handler(self): self.provider.get_output.return_value = "Test Output" - value = handler("fully-qualified-stack-name::SomeOutput", - provider=self.provider, context=self.context) + value = XrefLookup.handle("fully-qualified-stack-name::SomeOutput", + provider=self.provider, + context=self.context) self.assertEqual(value, "Test Output") self.assertEqual(self.context.get_fqn.call_count, 0) args = self.provider.get_output.call_args diff --git a/stacker/tests/lookups/test_registry.py b/stacker/tests/lookups/test_registry.py index 7ef338013..1dc0b41f1 100644 --- a/stacker/tests/lookups/test_registry.py +++ b/stacker/tests/lookups/test_registry.py @@ -3,19 +3,16 @@ from __future__ import absolute_import import unittest -from mock import patch, MagicMock +from mock import MagicMock from stacker.exceptions import ( UnknownLookupType, FailedVariableLookup, ) -from stacker.lookups.registry import ( - LOOKUP_HANDLERS, - resolve_lookups, -) +from stacker.lookups.registry import LOOKUP_HANDLERS -from stacker.variables import Variable +from stacker.variables import Variable, VariableValueLookup from ..factories import ( mock_context, @@ -43,31 +40,32 @@ def test_autoloaded_lookup_handlers(self): ) def test_resolve_lookups_string_unknown_lookup(self): - variable = Variable("MyVar", "${bad_lookup foo}") - with self.assertRaises(UnknownLookupType): - resolve_lookups(variable, self.ctx, self.provider) + Variable("MyVar", "${bad_lookup foo}") def test_resolve_lookups_list_unknown_lookup(self): - variable = Variable( - "MyVar", [ - "${bad_lookup foo}", "random string", - ] - ) - with self.assertRaises(UnknownLookupType): - resolve_lookups(variable, self.ctx, self.provider) + Variable( + "MyVar", [ + "${bad_lookup foo}", "random string", + ] + ) def resolve_lookups_with_output_handler_raise_valueerror(self, variable): """Mock output handler to throw ValueError, then run resolve_lookups on the given variable. """ mock_handler = MagicMock(side_effect=ValueError("Error")) - with patch.dict(LOOKUP_HANDLERS, {"output": mock_handler}): - with self.assertRaises(FailedVariableLookup) as cm: - resolve_lookups(variable, self.ctx, self.provider) - self.assertIsInstance(cm.exception.error, ValueError) + # find the only lookup in the variable + for value in variable._value: + if isinstance(value, VariableValueLookup): + value.handler = mock_handler + + with self.assertRaises(FailedVariableLookup) as cm: + variable.resolve(self.ctx, self.provider) + + self.assertIsInstance(cm.exception.error, ValueError) def test_resolve_lookups_string_failed_variable_lookup(self): variable = Variable("MyVar", "${output foo::bar}") diff --git a/stacker/tests/test_stack.py b/stacker/tests/test_stack.py index a51ccbc0b..c1bba0156 100644 --- a/stacker/tests/test_stack.py +++ b/stacker/tests/test_stack.py @@ -4,6 +4,7 @@ from mock import MagicMock import unittest +from stacker.lookups import register_lookup_handler from stacker.context import Context from stacker.config import Config from stacker.stack import Stack @@ -20,6 +21,7 @@ def setUp(self): definition=generate_definition("vpc", 1), context=self.context, ) + register_lookup_handler("noop", lambda **kwargs: "test") def test_stack_requires(self): definition = generate_definition( diff --git a/stacker/tests/test_variables.py b/stacker/tests/test_variables.py index 08daf0327..2b1acbc55 100644 --- a/stacker/tests/test_variables.py +++ b/stacker/tests/test_variables.py @@ -13,7 +13,7 @@ from stacker.stack import Stack -from .factories import mock_lookup, generate_definition +from .factories import generate_definition class TestVariables(unittest.TestCase): @@ -24,27 +24,11 @@ def setUp(self): def test_variable_replace_no_lookups(self): var = Variable("Param1", "2") - self.assertEqual(len(var.lookups), 0) - resolved_lookups = { - mock_lookup("fakeStack::FakeOutput", "output"): "resolved", - } - var.replace(resolved_lookups) - self.assertEqual(var.value, "2") - - def test_variable_resolve_no_lookups(self): - var = Variable("Param1", "2") - self.assertEqual(len(var.lookups), 0) - var.resolve(self.context, self.provider) - self.assertTrue(var.resolved) self.assertEqual(var.value, "2") def test_variable_replace_simple_lookup(self): var = Variable("Param1", "${output fakeStack::FakeOutput}") - self.assertEqual(len(var.lookups), 1) - resolved_lookups = { - mock_lookup("fakeStack::FakeOutput", "output"): "resolved", - } - var.replace(resolved_lookups) + var._value._resolve("resolved") self.assertEqual(var.value, "resolved") def test_variable_resolve_simple_lookup(self): @@ -59,32 +43,26 @@ def test_variable_resolve_simple_lookup(self): self.context.get_stack.return_value = stack var = Variable("Param1", "${output fakeStack::FakeOutput}") - self.assertEqual(len(var.lookups), 1) var.resolve(self.context, self.provider) self.assertTrue(var.resolved) self.assertEqual(var.value, "resolved") - self.assertEqual(len(var.lookups), 0) def test_variable_resolve_default_lookup_empty(self): var = Variable("Param1", "${default fakeStack::}") - self.assertEqual(len(var.lookups), 1) var.resolve(self.context, self.provider) self.assertTrue(var.resolved) self.assertEqual(var.value, "") - self.assertEqual(len(var.lookups), 0) def test_variable_replace_multiple_lookups_string(self): var = Variable( "Param1", - "url://${output fakeStack::FakeOutput}@" - "${output fakeStack::FakeOutput2}", + "url://" # 0 + "${output fakeStack::FakeOutput}" # 1 + "@" # 2 + "${output fakeStack::FakeOutput2}", # 3 ) - self.assertEqual(len(var.lookups), 2) - resolved_lookups = { - mock_lookup("fakeStack::FakeOutput", "output"): "resolved", - mock_lookup("fakeStack::FakeOutput2", "output"): "resolved2", - } - var.replace(resolved_lookups) + var._value[1]._resolve("resolved") + var._value[3]._resolve("resolved2") self.assertEqual(var.value, "url://resolved@resolved2") def test_variable_resolve_multiple_lookups_string(self): @@ -93,7 +71,6 @@ def test_variable_resolve_multiple_lookups_string(self): "url://${output fakeStack::FakeOutput}@" "${output fakeStack::FakeOutput2}", ) - self.assertEqual(len(var.lookups), 2) stack = Stack( definition=generate_definition("vpc", 1), @@ -110,23 +87,17 @@ def test_variable_resolve_multiple_lookups_string(self): def test_variable_replace_no_lookups_list(self): var = Variable("Param1", ["something", "here"]) - self.assertEqual(len(var.lookups), 0) - resolved_lookups = { - mock_lookup("fakeStack::FakeOutput", "output"): "resolved", - } - var.replace(resolved_lookups) self.assertEqual(var.value, ["something", "here"]) def test_variable_replace_lookups_list(self): - value = ["something", "${output fakeStack::FakeOutput}", - "${output fakeStack::FakeOutput2}"] + value = ["something", # 0 + "${output fakeStack::FakeOutput}", # 1 + "${output fakeStack::FakeOutput2}" # 2 + ] var = Variable("Param1", value) - self.assertEqual(len(var.lookups), 2) - resolved_lookups = { - mock_lookup("fakeStack::FakeOutput", "output"): "resolved", - mock_lookup("fakeStack::FakeOutput2", "output"): "resolved2", - } - var.replace(resolved_lookups) + + var._value[1]._resolve("resolved") + var._value[2]._resolve("resolved2") self.assertEqual(var.value, ["something", "resolved", "resolved2"]) def test_variable_replace_lookups_dict(self): @@ -135,12 +106,8 @@ def test_variable_replace_lookups_dict(self): "other": "${output fakeStack::FakeOutput2}", } var = Variable("Param1", value) - self.assertEqual(len(var.lookups), 2) - resolved_lookups = { - mock_lookup("fakeStack::FakeOutput", "output"): "resolved", - mock_lookup("fakeStack::FakeOutput2", "output"): "resolved2", - } - var.replace(resolved_lookups) + var._value["something"]._resolve("resolved") + var._value["other"]._resolve("resolved2") self.assertEqual(var.value, {"something": "resolved", "other": "resolved2"}) @@ -157,13 +124,10 @@ def test_variable_replace_lookups_mixed(self): }, } var = Variable("Param1", value) - self.assertEqual(len(var.lookups), 3) - resolved_lookups = { - mock_lookup("fakeStack::FakeOutput", "output"): "resolved", - mock_lookup("fakeStack::FakeOutput2", "output"): "resolved2", - mock_lookup("fakeStack::FakeOutput3", "output"): "resolved3", - } - var.replace(resolved_lookups) + var._value["something"][0]._resolve("resolved") + var._value["here"]["other"]._resolve("resolved2") + var._value["here"]["same"]._resolve("resolved") + var._value["here"]["mixed"][1]._resolve("resolved3") self.assertEqual(var.value, { "something": [ "resolved", @@ -194,11 +158,6 @@ def mock_handler(value, context, provider, **kwargs): "Param1", "${lookup ${lookup ${output fakeStack::FakeOutput}}}", ) - self.assertEqual( - len(var.lookups), - 1, - "should only parse out the first complete lookup first", - ) var.resolve(self.context, self.provider) self.assertTrue(var.resolved) self.assertEqual(var.value, "looked up: looked up: resolved") diff --git a/stacker/variables.py b/stacker/variables.py index c27cdb899..5f49b2074 100644 --- a/stacker/variables.py +++ b/stacker/variables.py @@ -1,15 +1,17 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import division + +import re + from past.builtins import basestring from builtins import object from string import Template -from .exceptions import InvalidLookupCombination -from .lookups import ( - extract_lookups, - resolve_lookups, -) +from .exceptions import InvalidLookupCombination, UnresolvedVariable, \ + UnknownLookupType, FailedVariableLookup, FailedLookup, \ + UnresolvedVariableValue, InvalidLookupConcatenation +from .lookups.registry import LOOKUP_HANDLERS class LookupTemplate(Template): @@ -18,53 +20,6 @@ class LookupTemplate(Template): idpattern = r'[_a-z][^\$\{\}]*' -def resolve_string(value, replacements): - """Resolve any lookups within a string. - - Args: - value (str): string value we're resolving lookups within - replacements (dict): resolved lookup values - - Returns: - str: value with any lookups resolved - - """ - lookups = extract_lookups(value) - for lookup in lookups: - lookup_value = replacements.get(lookup.raw) - if not isinstance(lookup_value, basestring): - if len(lookups) > 1: - raise InvalidLookupCombination(lookup, lookups, value) - return lookup_value - # we use safe_substitute to support resolving nested lookups - return LookupTemplate(value).safe_substitute(replacements) - - -def resolve(value, replacements): - """Recursively resolve any lookups within the data structure. - - Args: - value (Union[str, list, dict]): a structure that contains lookups - replacements: resolved lookup values - - Returns: - Union[str, list, dict]: value passed in with lookup values resolved - - """ - if isinstance(value, basestring): - return resolve_string(value, replacements) - elif isinstance(value, list): - resolved = [] - for v in value: - resolved.append(resolve(v, replacements)) - return resolved - elif isinstance(value, dict): - for key, v in value.items(): - value[key] = resolve(v, replacements) - return value - return value - - def resolve_variables(variables, context, provider): """Given a list of variables, resolve all of them. @@ -81,54 +36,37 @@ def resolve_variables(variables, context, provider): class Variable(object): - """Represents a variable passed to a stack. Args: name (str): Name of the variable - value (str): Initial value of the variable from the config - + value (any): Initial value of the variable from the config (str, list, + dict) """ def __init__(self, name, value): self.name = name - self._value = value - self._resolved_value = None - - @property - def lookups(self): - """Return any lookups within the value""" - return extract_lookups(self.value) - - @property - def needs_resolution(self): - """Return True if the value has any lookups that need resolving.""" - if self.lookups: - return True - return False + self._raw_value = value + self._value = VariableValue.parse(value) @property def value(self): """Return the current value of the Variable. - - `_resolved_value` takes precedence over `_value`. - """ - if self._resolved_value is not None: - return self._resolved_value - else: - return self._value + try: + return self._value.value() + except UnresolvedVariableValue: + raise UnresolvedVariable("", self) + except InvalidLookupConcatenation as e: + raise InvalidLookupCombination(e.lookup, e.lookups, self) @property def resolved(self): """Boolean for whether the Variable has been resolved. Variables only need to be resolved if they contain lookups. - """ - if self.needs_resolution: - return self._resolved_value is not None - return True + return self._value.resolved() def resolve(self, context, provider): """Recursively resolve any lookups with the Variable. @@ -140,21 +78,362 @@ def resolve(self, context, provider): the base provider """ + try: + self._value.resolve(context, provider) + except FailedLookup as e: + raise FailedVariableLookup(self.name, e.lookup, e.error) - while self.lookups: - resolved_lookups = resolve_lookups(self, context, provider) - self.replace(resolved_lookups) + def dependencies(self): + """ + Returns: + Set[str]: Stack names that this variable depends on + """ + return self._value.dependencies() - def replace(self, resolved_lookups): - """Replace lookups in the Variable with their resolved values. - Args: - resolved_lookups (dict): dict of :class:`stacker.lookups.Lookup` -> - resolved value. +class VariableValue(object): + """ + Abstract Syntax Tree base object to parse the value for a variable + """ + def value(self): + return NotImplementedError() + + def __iter__(self): + return NotImplementedError() + + def resolved(self): + """ + Returns: + bool: Whether value() will not raise an error + """ + return NotImplementedError() + + def resolve(self, context, provider): + pass + + def dependencies(self): + return set() + + def simplified(self): + """ + Return a simplified version of the Value. + This can be used to e.g. concatenate two literals in to one literal, or + to flatten nested Concatenations + + Returns: + VariableValue + """ + return self + + @classmethod + def parse(cls, input_object): + if isinstance(input_object, list): + return VariableValueList.parse(input_object) + elif isinstance(input_object, dict): + return VariableValueDict.parse(input_object) + elif not isinstance(input_object, basestring): + return VariableValueLiteral(input_object) + # else: # str + + tokens = VariableValueConcatenation([ + VariableValueLiteral(t) + for t in re.split(r'(\$\{|\}|\s+)', input_object) + ]) + + opener = '${' + closer = '}' + + while True: + last_open = None + next_close = None + for i, t in enumerate(tokens): + if not isinstance(t, VariableValueLiteral): + continue + + if t.value() == opener: + last_open = i + next_close = None + if last_open is not None and \ + t.value() == closer and \ + next_close is None: + next_close = i + + if next_close is not None: + lookup_data = VariableValueConcatenation( + tokens[(last_open + len(opener) + 1):next_close] + ) + lookup = VariableValueLookup( + lookup_name=tokens[last_open + 1], + lookup_data=lookup_data, + ) + tokens[last_open:(next_close + 1)] = [lookup] + else: + break + + tokens = tokens.simplified() + + return tokens + + +class VariableValueLiteral(VariableValue): + def __init__(self, value): + self._value = value + + def value(self): + return self._value + + def __iter__(self): + yield self + + def resolved(self): + return True + + def __repr__(self): + return "Literal<{}>".format(repr(self._value)) + + +class VariableValueList(VariableValue, list): + @classmethod + def parse(cls, input_object): + acc = [ + VariableValue.parse(obj) + for obj in input_object + ] + return cls(acc) + + def value(self): + return [ + item.value() + for item in self + ] + + def resolved(self): + accumulator = True + for item in self: + accumulator = accumulator and item.resolved() + return accumulator + + def __repr__(self): + return "List[{}]".format(', '.join([repr(value) for value in self])) + + def __iter__(self): + return list.__iter__(self) + + def resolve(self, context, provider): + for item in self: + item.resolve(context, provider) + + def dependencies(self): + deps = set() + for item in self: + deps.update(item.dependencies()) + return deps + + def simplified(self): + return [ + item.simplified() + for item in self + ] + + +class VariableValueDict(VariableValue, dict): + @classmethod + def parse(cls, input_object): + acc = { + k: VariableValue.parse(v) + for k, v in input_object.items() + } + return cls(acc) + + def value(self): + return { + k: v.value() + for k, v in self.items() + } + + def resolved(self): + accumulator = True + for item in self.values(): + accumulator = accumulator and item.resolved() + return accumulator + + def __repr__(self): + return "Dict[{}]".format(', '.join([ + "{}={}".format(k, repr(v)) for k, v in self.items() + ])) + + def __iter__(self): + return dict.__iter__(self) + + def resolve(self, context, provider): + for item in self.values(): + item.resolve(context, provider) + + def dependencies(self): + deps = set() + for item in self.values(): + deps.update(item.dependencies()) + return deps + + def simplified(self): + return { + k: v.simplified() + for k, v in self.items() + } + + +class VariableValueConcatenation(VariableValue, list): + def value(self): + if len(self) == 1: + return self[0].value() + values = [] + for value in self: + resolved_value = value.value() + if not isinstance(resolved_value, basestring): + raise InvalidLookupConcatenation(value, self) + values.append(resolved_value) + return ''.join(values) + + def __iter__(self): + return list.__iter__(self) + + def resolved(self): + accumulator = True + for item in self: + accumulator = accumulator and item.resolved() + return accumulator + + def __repr__(self): + return "Concat[{}]".format(', '.join([repr(value) for value in self])) + + def resolve(self, context, provider): + for value in self: + value.resolve(context, provider) + + def dependencies(self): + deps = set() + for item in self: + deps.update(item.dependencies()) + return deps + + def simplified(self): + concat = [] + for item in self: + if isinstance(item, VariableValueLiteral) and \ + item.value() == '': + pass + + elif isinstance(item, VariableValueLiteral) and \ + len(concat) > 0 and \ + isinstance(concat[-1], VariableValueLiteral): + # Join the literals together + concat[-1] = VariableValueLiteral( + concat[-1].value() + item.value() + ) + + elif isinstance(item, VariableValueConcatenation): + # Flatten concatenations + concat.extend(item.simplified()) + + else: + concat.append(item.simplified()) + + if len(concat) == 0: + return VariableValueLiteral('') + elif len(concat) == 1: + return concat[0] + else: + return VariableValueConcatenation(concat) + + +class VariableValueLookup(VariableValue): + def __init__(self, lookup_name, lookup_data, handler=None): + """ + Args: + lookup_name (basestring): Name of the invoked lookup + lookup_data (VariableValue): Data portion of the lookup """ - replacements = {} - for lookup, value in resolved_lookups.items(): - replacements[lookup.raw] = value + self._resolved = False + self._value = None + + self.lookup_name = lookup_name + + if isinstance(lookup_data, basestring): + lookup_data = VariableValueLiteral(lookup_data) + self.lookup_data = lookup_data - self._resolved_value = resolve(self.value, replacements) + if handler is None: + lookup_name_resolved = lookup_name.value() + try: + handler = LOOKUP_HANDLERS[lookup_name_resolved] + except KeyError: + raise UnknownLookupType(lookup_name_resolved) + self.handler = handler + + def resolve(self, context, provider): + self.lookup_data.resolve(context, provider) + try: + if type(self.handler) == type: + # Hander is a new-style handler + result = self.handler.handle( + value=self.lookup_data.value(), + context=context, + provider=provider + ) + else: + result = self.handler( + value=self.lookup_data.value(), + context=context, + provider=provider + ) + self._resolve(result) + except Exception as e: + raise FailedLookup(self, e) + + def _resolve(self, value): + self._value = value + self._resolved = True + + def dependencies(self): + if type(self.handler) == type: + return self.handler.dependencies(self.lookup_data) + else: + return set() + + def value(self): + if self._resolved: + return self._value + else: + raise UnresolvedVariableValue(self) + + def __iter__(self): + yield self + + def resolved(self): + return self._resolved + + def __repr__(self): + if self._resolved: + return "Lookup<{r} ({t} {d})>".format( + r=self._value, + t=self.lookup_name, + d=repr(self.lookup_data), + ) + else: + return "Lookup<{t} {d}>".format( + t=self.lookup_name, + d=repr(self.lookup_data), + ) + + def __str__(self): + return "${{{type} {data}}}".format( + type=self.lookup_name.value(), + data=self.lookup_data.value(), + ) + + def simplified(self): + return VariableValueLookup( + lookup_name=self.lookup_name, + lookup_data=self.lookup_data.simplified(), + )